0 branches 0 tags
92
Fix coroutine trimming calculations
jonroach 5 Jun 2026 16:48
1 contributor
1593 lines39.7 KB
Newer
Older
Text Raw Blame
  • Delete
-
+
commited
{line.log.rev}
 on
9 months ago
4
1
 #include "coroutine.h"
2
 #include <assert.h>
3
 #include <setjmp.h>
4
 #include <stdbool.h>
5
 #include <stddef.h>
6 months ago
6
 #include <stdio.h>
1 weeks ago
7
 #include <stdlib.h>
8 months ago
8
 #include "cor_platform.h"
9 months ago
4
9
7 months ago
10
 // see CPython again, this time from ctypes.h
11
 #if (defined (__SVR4) && defined (__sun)) || defined(COROUTINE_HAVE_ALLOCA_H)
12
 # include <alloca.h>
13
 #elif defined(MS_WIN32)
14
 # include <malloc.h>
15
 #endif
9 months ago
4
16
7 months ago
17
 /* If the system does not define alloca(), we have to hope for a compiler builtin. */
18
 #ifndef alloca
19
 # if defined __GNUC__ || (__clang_major__ >= 4)
20
 # define alloca __builtin_alloca
21
 # else
22
 # error "Could not define alloca() on your platform."
23
 # endif
24
 #endif
25
4 months ago
26
 typedef struct Coroutines Coroutines;
27
Last month
28
 static void Coroutine_NS(RunNext)(void);
29
 static Coroutine_Err Coroutine_NS(Continue_)(Coroutines *cors, Coroutine *cor, void *value, bool early);
Last month
30
 static uintptr_t StackTopNow(void);
9 months ago
31
4 months ago
32
 #ifndef NDEBUG
33
 // In debug builds, use the built-in assert
34
 #define MyAssert assert
35
 #else
36
 #if 1
37
 // In non-debug builds, normally use this - all the asserts are disabled
38
 #define MyAssert(cond)
39
 #else
40
 // In non-debug builds with stack problems, you can use this.
41
 // This activates all the asserts, and gives a line to put a
42
 // breakpoint in your debugger.
43
 static void _MyAssert(bool cond, char const *msg)
44
 {
45
 if (!cond){
46
 fputs("Assertion failed: ", stdout);
47
 fputs(msg, stdout);
48
 fputs("\n", stdout);
49
 }
50
 }
51
 #define MyAssert(cond) _MyAssert(cond, #cond)
52
 #endif
53
 #endif
54
1 weeks ago
55
 static inline ptrdiff_t
56
 StackPointerDiff
57
 (
58
 unsigned char *a,
59
 unsigned char *b
60
 ){
61
 #if COROUTINE_STACK_GROWS_UP
62
 return a - b;
63
 #else
64
 return b - a;
65
 #endif
66
 }
67
68
 static inline unsigned char *
69
 StackPointerAdd
70
 (
71
 unsigned char *a,
72
 ptrdiff_t b
73
 ){
74
 #if COROUTINE_STACK_GROWS_UP
75
 return a + b;
76
 #else
77
 return a - b;
78
 #endif
79
 }
80
81
 // Unused...
82
 // static inline unsigned char *
83
 // StackBaseEnd
84
 // (
85
 // unsigned char *memlo,
86
 // unsigned char *memhi
87
 // ){
88
 // #if COROUTINE_STACK_GROWS_UP
89
 // (void)memhi;
90
 // return memlo;
91
 // #else
92
 // (void)memlo;
93
 // return memhi-1;
94
 // #endif
95
 // }
96
 // ...unused
97
98
 static inline unsigned char *
99
 StackLimitEnd
100
 (
101
 unsigned char *memlo,
102
 unsigned char *memhi
103
 ){
104
 #if COROUTINE_STACK_GROWS_UP
105
 (void)memlo;
106
 return memhi;
107
 #else
108
 (void)memhi;
109
 return memlo-1;
110
 #endif
111
 }
112
4 months ago
113
 #define CHECK_SYSTEM_RUNNING \
114
 if (!g_c){ \
115
 return Coroutine_Err_SystemNotRunning; \
116
 }
117
 #define CHECK_SYSTEM_NOT_RUNNING \
118
 if (g_c){ \
119
 return Coroutine_Err_SystemRunning; \
120
 }
121
 #define CHECK_COROUTINE_THREAD \
122
 if (cor->coroutines != g_c){ \
123
 return Coroutine_Err_CoroutineFromWrongThread; \
124
 }
125
 #define CHECK_NO_COROUTINE_RUNNING \
126
 if (g_c->state != Coroutines_Started){ \
127
 return Coroutine_Err_ACoroutineIsAlreadyRunning; \
128
 }
129
 #define CHECK_STACK_OVERRUN \
130
 { \
Last month
131
 Coroutine_Err err = Coroutine_NS(StackHasOverrun)(); \
4 months ago
132
 if (err){ \
133
 return err; \
134
 } \
135
 } while (0);
136
Last month
137
138
 static inline void ready_jmp_buf(jmp_buf buf) {
139
 #if defined(_M_X64) || defined(_M_ARM64)
140
 // Win64:
141
 // Set Frame to 0 on Windows 64 bit to prevent C++ stack unwinding in longjmp().
142
 // Win32:
143
 // Doesn't do this, so only needed on the 2 64 bit Windows versions
144
 ((_JUMP_BUFFER*)buf)->Frame = 0;
Last month
145
 #else
146
 (void)buf;
Last month
147
 #endif
148
 }
149
9 months ago
4
150
 ///////////////////////////////////////////////////////////////////////////////
151
 // 2-way linked lists...
152
 //
9 months ago
153
 // Brought inline here to avoid namespace polution
9 months ago
4
154
 ///////////////////////////////////////////////////////////////////////////////
155
156
 typedef struct List_Link List_Link;
157
 struct List_Link {
158
 List_Link *next;
159
 List_Link *prev;
160
 };
161
162
 typedef struct List_Head List_Head;
163
 struct List_Head {
164
 union {
165
 struct {
166
 List_Link link;
167
 List_Link *filler;
168
 } fwd;
169
 struct {
170
 List_Link *filler;
171
 List_Link link;
172
 } back;
173
 };
174
 };
175
9 months ago
176
177
 static inline bool List_IsEmpty(
178
 const List_Head *list
179
 ){
9 months ago
4
180
 return list->fwd.link.next == &list->back.link;
181
 }
182
9 months ago
183
184
 static inline List_Link *List_GetHead(
185
 const List_Head *list
186
 ){
9 months ago
4
187
 return List_IsEmpty(list) ? NULL : list->fwd.link.next;
188
 }
9 months ago
189
190
6 months ago
191
 static inline List_Link *List_Begin(
192
 const List_Head *list
193
 ){
194
 return list->fwd.link.next;
195
 }
196
197
198
 static inline bool Link_NextIsLink(
199
 const List_Link *link
200
 ){
201
 return link->next != NULL;
202
 }
203
204
205
 static inline List_Link *Link_Next(
206
 List_Link *link
207
 ){
208
 return link->next;
209
 }
210
211
212
 static inline bool Link_PrevIsLink(
213
 const List_Link *link
214
 ){
215
 return link->prev != NULL;
216
 }
217
218
219
 static inline List_Link *Link_Prev(
220
 List_Link *link
221
 ){
222
 return link->prev;
223
 }
224
1 weeks ago
225
 // Unused...
226
 // static inline List_Link *List_GetTail(
227
 // const List_Head *list
228
 // ){
229
 // return List_IsEmpty(list) ? NULL : list->back.link.prev;
230
 // }
231
 // ...unused
9 months ago
232
233
9 months ago
4
234
 #define OFFSETOF(Container, Field) ((char *)&((Container *)4)->Field - (char *)(Container *)4)
235
 #define List_Link_Container(Container, Link, link) ((Container *)((char *)(link) - OFFSETOF(Container, Link)))
236
9 months ago
237
Last month
238
 static inline void
239
 List_Init(
9 months ago
240
 List_Head *list
241
 ){
9 months ago
4
242
 list->fwd.link.next = &list->back.link;
243
 list->fwd.link.prev = NULL;
244
 list->back.link.prev = &list->fwd.link;
245
 }
246
9 months ago
247
Last month
248
 static inline void
249
 Link_AddAfter(
6 months ago
250
 List_Link *link,
251
 List_Link *after
252
 ){
253
 link->next = after->next;
254
 link->prev = after;
255
 after->next->prev = link;
256
 after->next = link;
257
 }
258
259
Last month
260
 static inline void
261
 List_AddHead(
9 months ago
262
 List_Head *list,
263
 List_Link *link
264
 ){
6 months ago
265
 Link_AddAfter(link, &list->fwd.link);
9 months ago
4
266
 }
267
9 months ago
268
Last month
269
 static inline void
270
 Link_AddBefore(
6 months ago
271
 List_Link *link,
272
 List_Link *before
273
 ){
274
 link->prev = before->prev;
275
 link->next = before;
276
 before->prev->next = link;
277
 before->prev = link;
278
 }
279
280
Last month
281
 static inline void
282
 List_AddTail(
9 months ago
283
 List_Head *list,
284
 List_Link *link
285
 ){
6 months ago
286
 Link_AddBefore(link, &list->back.link);
9 months ago
4
287
 }
288
9 months ago
289
Last month
290
 static inline void
291
 Link_Remove(
9 months ago
292
 List_Link *link
293
 ){
9 months ago
4
294
 link->prev->next = link->next;
295
 link->next->prev = link->prev;
296
 }
297
298
 ///////////////////////////////////////////////////////////////////////////////
299
 // ...2-way linked lists
300
 ///////////////////////////////////////////////////////////////////////////////
301
302
 enum {
303
 Coroutines_Starting,
304
 Coroutines_Started,
305
 Coroutines_Active,
306
 Coroutines_Stopping
307
 };
308
309
 enum {
310
 Chunk_Initial,
6 months ago
311
 Chunk_Split,
9 months ago
4
312
 Chunk_Enter
313
 };
314
9 months ago
315
 typedef enum Coroutine_State {
Last month
316
 Coroutine_Free,
317
 Coroutine_Idle,
318
 Coroutine_Running,
319
 Coroutine_Waiting,
320
 Coroutine_Complete
9 months ago
321
 } Coroutine_State;
9 months ago
4
322
323
 enum {
324
 Coroutines_Init,
325
 Coroutines_AllocatedChunk,
326
 Coroutines_CoroutineComplete,
327
 };
328
329
 struct Coroutine {
1 weeks ago
330
 size_t min_size;
331
 size_t min_headroom;
7 months ago
332
 Coroutines *coroutines; // so can work with it off-thread
333
 List_Link link; // for whichever list it's on
6 months ago
334
 List_Link all_link; // list of all Coroutines
7 months ago
335
 jmp_buf buf; // how to get back to it
1 weeks ago
336
 unsigned char *base; // where the base of this Coroutine's stack is (= previous block's limit)
337
 unsigned char *limit; // where the limit of this Coroutine's stack is (= next block's base)
7 months ago
338
 unsigned char *guard; // where the stack overrun guard is
339
 Coroutine_Start start; // entry point
340
 void *entry_param; // to pass to start
341
 void *value; // yielded/returned
342
 unsigned char *stack_top; // recorded at yield
9 months ago
343
 Coroutine_State state;
1 weeks ago
344
345
 int sequence;
9 months ago
4
346
 };
347
348
 struct Coroutines {
8 months ago
349
 _Cor_Mutex mutex;
Last month
350
 jmp_buf controller; // to return from Coroutine_NS(Run)
9 months ago
351
 jmp_buf chunk_allocated;// for chunk allocation
1 weeks ago
352
 size_t size_to_retain; // for Chunk_Split
9 months ago
4
353
354
 // singletons
355
 Coroutine *tip; // top of stack chunk
356
 Coroutine *active; // currently running coroutine
Last month
357
 Coroutine *primary; // Coroutine_NS(Run) coroutine
7 months ago
358
 unsigned char *stack_limit; // when not NULL, where the stack finishes
9 months ago
4
359
1 weeks ago
360
 Coroutine *spare; // spare struct Coroutine (instead of having to malloc & fail)
361
9 months ago
4
362
 // lists
6 months ago
363
 List_Head all; // all Coroutines (in address order)
364
 List_Head free; // free Coroutines
9 months ago
4
365
 List_Head inactive; // idle or complete
366
 List_Head runable; // running or waiting to run
367
 List_Head waiting; // yielded / waiting to run
8 months ago
368
 _Cor_Mutex waiting_mutex;
9 months ago
4
369
1 weeks ago
370
 Coroutine *root; // The coroutine for the thread which started Coroutines
371
9 months ago
372
 // Summary of the system
373
 Coroutine_Report report;
374
9 months ago
4
375
 // state
376
 char state;
1 weeks ago
377
378
 int sequence;
9 months ago
4
379
 };
380
6 months ago
381
 _Cor_thread_local Coroutines *g_c;
5 months ago
382
 _Cor_thread_local unsigned char *g_stack_limit;
9 months ago
4
383
6 months ago
384
 static void ReserveStackSpace(Coroutines *cors, Coroutine *parent, size_t chunk_size, unsigned char *childs_limit);
6 months ago
385
 static void stack_chunk_base(Coroutines *cors, Coroutine *parent, unsigned char *prev_limit, unsigned char *limit);
9 months ago
4
386
9 months ago
387
1 weeks ago
388
 static size_t
389
 Coroutine_Size
390
 (
391
 Coroutine *cor
392
 ){
393
 if (cor->limit){
394
 return (size_t)StackPointerDiff(cor->limit, cor->base);
395
 } else {
396
 return SIZE_MAX;
397
 }
398
 }
399
400
401
 static size_t
1 weeks ago
402
 TrimmableSize(
403
 Coroutine *cor
404
 ){
1 weeks ago
405
 size_t current_used = StackPointerDiff(cor->stack_top, cor->base);
406
 size_t min_size = current_used + cor->min_headroom;
407
 if (min_size < cor->min_size){
408
 min_size = cor->min_size;
1 weeks ago
409
 }
410
411
 if (cor->limit){
1 weeks ago
412
 if (Coroutine_Size(cor) < min_size + COROUTINE_MINIMUM_STACK_SIZE){
1 weeks ago
413
 return 0;
414
 }
415
 }
416
 return min_size;
417
 }
418
419
7 months ago
420
 #define GUARD_PATTERN_SIZE (4)
1 weeks ago
421
 /// @brief Checks whether a guard pattern is OK
422
 /// @param guard The address of the stack base end of the pattern
423
 /// @return true pattern is ok; false pattern is broken
Last month
424
 static inline bool
425
 Guard_Pattern_OK(
9 months ago
426
 unsigned char *guard
427
 ){
7 months ago
428
 return !guard ||
1 weeks ago
429
 (*StackPointerAdd(guard, 0) == 0xde &&
430
 *StackPointerAdd(guard, 1) == 0xad &&
431
 *StackPointerAdd(guard, 2) == 0xbe &&
432
 *StackPointerAdd(guard, 3) == 0xef);
9 months ago
433
 }
434
435
1 weeks ago
436
 /// @brief Writes a guard pattern
437
 /// @param guard Where to write the guard pattern (stack base end)
Last month
438
 static inline void
439
 Apply_Guard(unsigned char *guard){
1 weeks ago
440
 *StackPointerAdd(guard, 0) = 0xde;
441
 *StackPointerAdd(guard, 1) = 0xad;
442
 *StackPointerAdd(guard, 2) = 0xbe;
443
 *StackPointerAdd(guard, 3) = 0xef;
7 months ago
444
 }
445
446
4 months ago
447
 #ifndef NDEBUG
1 weeks ago
448
 /// @brief Checks a list of Coroutines to see whether it's OK
449
 /// @param head The list to check
450
 /// @param state1 One of the states Coroutines in the list are allowed
451
 /// @param state2 One of the states Coroutines in the list are allowed
452
 /// @return Coroutine_OK the is OK; other the list is broken
Last month
453
 static Coroutine_Err
454
 CheckListIntegrity(
455
 List_Head *head,
456
 Coroutine_State state1,
457
 Coroutine_State state2
458
 ){
4 months ago
459
 for (List_Link *link = List_Begin(head); Link_NextIsLink(link); link = Link_Next(link)){
460
 Coroutine *candidate = List_Link_Container(Coroutine, link, link);
4 months ago
461
 if (candidate->coroutines != g_c){
462
 return Coroutine_Err_InternalInsistency;
463
 }
464
 if(candidate->state != state1 && candidate->state != state2){
465
 return Coroutine_Err_InternalInsistency;
466
 }
4 months ago
467
 bool found = false;
468
 for (List_Link *link = List_Begin(&g_c->all); Link_NextIsLink(link); link = Link_Next(link)){
469
 Coroutine *candidate2 = List_Link_Container(Coroutine, all_link, link);
470
 if (candidate == candidate2){
471
 found = true;
472
 }
473
 }
4 months ago
474
 if (!found){
475
 return Coroutine_Err_InternalInsistency;
476
 }
4 months ago
477
 }
4 months ago
478
 return Coroutine_OK;
4 months ago
479
 }
480
481
1 weeks ago
482
 /// @brief Check the integrity of this thread's coroutine system
483
 /// @return Coroutine_OK everything's OK; other something was wrong
Last month
484
 static Coroutine_Err
485
 Coroutine_NS(CheckIntegrity_)(
486
 void
487
 ){
4 months ago
488
 Coroutine_Err err;
Last month
489
 err = CheckListIntegrity(&g_c->free, Coroutine_Free, Coroutine_Free);
4 months ago
490
 if (err){
491
 return err;
492
 }
Last month
493
 err = CheckListIntegrity(&g_c->inactive, Coroutine_Idle, Coroutine_Complete);
4 months ago
494
 if (err){
495
 return err;
496
 }
Last month
497
 err = CheckListIntegrity(&g_c->runable, Coroutine_Running, Coroutine_Running);
4 months ago
498
 if (err){
499
 return err;
500
 }
Last month
501
 err = CheckListIntegrity(&g_c->waiting, Coroutine_Waiting, Coroutine_Waiting);
4 months ago
502
 return err;
4 months ago
503
 }
4 months ago
504
 #endif
505
506
1 weeks ago
507
 /// @brief Check whether the stack has overrun
508
 /// @return Coroutine_OK the stack hasn't; other it has
Last month
509
 static Coroutine_Err
510
 Coroutine_NS(StackHasOverrun)(
511
 void
512
 ){
Last month
513
 unsigned char *stack_top = (unsigned char *)StackTopNow();
6 months ago
514
 unsigned char *stack_limit = g_c ? g_c->stack_limit : NULL;
1 weeks ago
515
 if (stack_limit && StackPointerDiff(stack_limit, stack_top) < 0){
4 months ago
516
 // printf("top %p < limit %p\n", stack_top, stack_limit);
7 months ago
517
 // current stack top is beyond limit - we are overrunning NOW
4 months ago
518
 return Coroutine_Err_StackOverrun;
7 months ago
519
 }
6 months ago
520
 Coroutine *me = g_c ? g_c->active : NULL;
7 months ago
521
 if (!me){
4 months ago
522
 return Coroutine_OK;
7 months ago
523
 }
4 months ago
524
 #if COROUTINE_CHECK_INTEGRITY_ON_STACK_CHECK
525
 // Check all coroutines integrity
1 weeks ago
526
 Coroutine_Err err = Coroutine_NS(CheckIntegrity_)();
4 months ago
527
 if (err){
528
 return err;
529
 }
4 months ago
530
 #endif
7 months ago
531
 if (me->guard){
1 weeks ago
532
 if (StackPointerDiff(me->guard, stack_top) < 0){
533
 printf("Stack top beyond active stack limit\n");
534
 return Coroutine_Err_StackOverrun;
535
 }
536
 if (!Guard_Pattern_OK(me->guard)){
Last month
537
 printf("Guard pattern trampled\n");
1 weeks ago
538
 return Coroutine_Err_StackOverrun;
Last month
539
 }
7 months ago
540
 }
1 weeks ago
541
 return Coroutine_OK;
7 months ago
542
 }
543
Last month
544
 #ifndef NDEBUG
1 weeks ago
545
 /// @brief Check system integrity - does stack overrun check and internals check
546
 /// @return Coroutine_OK all is OK; other something was wrong
Last month
547
 Coroutine_Err
548
 Coroutine_NS(CheckIntegrity)(
549
 void
550
 ){
551
 Coroutine_Err err = Coroutine_NS(StackHasOverrun)();
Last month
552
 #if !COROUTINE_CHECK_INTEGRITY_ON_STACK_CHECK
553
 if (!err && g_c){
Last month
554
 err = Coroutine_NS(CheckIntegrity_)();
Last month
555
 }
556
 #endif
557
 return err;
558
 }
559
 #endif
7 months ago
560
Last month
561
Last month
562
 static void
563
 ReserveStackSpace(
6 months ago
564
 Coroutines *cors,
7 months ago
565
 Coroutine *parent,
6 months ago
566
 size_t chunk_size,
567
 unsigned char *childs_limit
9 months ago
568
 ){
7 months ago
569
 unsigned char *chunk_of_stack = alloca(chunk_size);
1 weeks ago
570
 unsigned char *limit = StackLimitEnd(chunk_of_stack, chunk_of_stack+chunk_size);
571
 unsigned char *guard = StackPointerAdd(limit, -GUARD_PATTERN_SIZE);
7 months ago
572
 #if COROUTINE_RECORD_LOWEST_HEADROOM
573
 for (size_t i = 0; i <= chunk_size-GUARD_PATTERN_SIZE; i += GUARD_PATTERN_SIZE){
1 weeks ago
574
 Apply_Guard(StackPointerAdd(guard, -i));
7 months ago
575
 }
576
 #else
1 weeks ago
577
 Apply_Guard(guard);
7 months ago
578
 #endif
6 months ago
579
 if (parent){
1 weeks ago
580
 parent->limit = limit;
581
 parent->guard = guard;
6 months ago
582
 }
1 weeks ago
583
 stack_chunk_base(cors, parent, limit, childs_limit);
9 months ago
4
584
 }
585
9 months ago
586
Last month
587
 static void
588
 stack_chunk_base(
6 months ago
589
 Coroutines *cors,
6 months ago
590
 Coroutine *parent,
6 months ago
591
 unsigned char *prev_limit,
592
 unsigned char *limit
9 months ago
593
 ){
1 weeks ago
594
 MyAssert(cors->spare);
595
 Coroutine *here = cors->spare;
596
 cors->spare = NULL;
597
 here->coroutines = cors;
598
 here->sequence = cors->sequence++;
599
 here->state = Coroutine_Free;
600
 here->base = prev_limit;
601
 here->limit = limit;
602
 here->stack_top = (unsigned char *)StackTopNow();
6 months ago
603
 if (limit){
1 weeks ago
604
 here->guard = StackPointerAdd(limit, -GUARD_PATTERN_SIZE);
6 months ago
605
 Apply_Guard(limit);
1 weeks ago
606
 } else {
607
 here->guard = NULL;
6 months ago
608
 }
609
6 months ago
610
 // insert into all list
611
 if (parent){
1 weeks ago
612
 Link_AddAfter(&here->all_link, &parent->all_link);
6 months ago
613
 } else {
1 weeks ago
614
 List_AddHead(&cors->all, &here->all_link);
6 months ago
615
 }
6 months ago
616
 // add to free list
1 weeks ago
617
 List_AddTail(&cors->free, &here->link);
6 months ago
618
619
 cors->report.coroutines_pool_size += 1;
620
1 weeks ago
621
 if (!cors->tip || !Link_NextIsLink(Link_Next(&here->all_link))){
622
 cors->tip = here;
6 months ago
623
 }
624
7 months ago
625
 for(;;){
1 weeks ago
626
 switch (setjmp(here->buf)) {
7 months ago
627
 case Chunk_Initial:
1 weeks ago
628
 ready_jmp_buf(here->buf);
629
 if (here->state == Coroutine_Free){
7 months ago
630
 // return to the coroutine allocator
6 months ago
631
 longjmp(cors->chunk_allocated, 1);
7 months ago
632
 } else {
1 weeks ago
633
 MyAssert(here->state == Coroutine_Complete);
7 months ago
634
 // we finish here to ensure the setjmp is redone
1 weeks ago
635
 if (cors->primary == here) {
Last month
636
 // if primary coroutine - return to Coroutine_NS(Run)
6 months ago
637
 longjmp(cors->controller, Coroutines_CoroutineComplete);
7 months ago
638
 }
6 months ago
639
 _Cor_Mutex_Unlock(&cors->mutex);
Last month
640
 Coroutine_NS(RunNext)();
7 months ago
641
 }
3 months ago
642
 MyAssert(false);
643
 break;
6 months ago
644
 case Chunk_Split:
1 weeks ago
645
 // Request to split this idle block into two
646
 // g_c->size_to_retain will be set to our shorter size
647
 ReserveStackSpace(here->coroutines, here, g_c->size_to_retain, here->limit);
4 months ago
648
 MyAssert(false);
3 months ago
649
 break;
7 months ago
650
 case Chunk_Enter:
651
 // request to start a coroutine (ie use the chunk for a coroutine)
652
 // arrive here with mutex locked
1 weeks ago
653
 MyAssert(here->state == Coroutine_Running);
654
 here->coroutines->active = here;
6 months ago
655
 _Cor_Mutex_Unlock(&cors->mutex);
1 weeks ago
656
 here->value = here->start(here->entry_param);
9 months ago
657
7 months ago
658
 // check the guard
1 weeks ago
659
 MyAssert(Guard_Pattern_OK(here->guard));
9 months ago
660
1 weeks ago
661
 here->stack_top = (unsigned char *)StackTopNow();
662
 _Cor_Mutex_Lock(&here->coroutines->mutex);
663
 here->coroutines->active = NULL;
664
 MyAssert(here->state == Coroutine_Running);
665
 Link_Remove(&here->link);
666
 here->state = Coroutine_Complete;
667
 List_AddTail(&here->coroutines->inactive, &here->link);
7 months ago
668
 // Coroutine has completed
669
 // Loop round to redo the setjmp() - if this coroutine yielded, then the setjmp will
670
 // need reseting
3 months ago
671
 break;
9 months ago
4
672
 }
673
 }
674
 }
675
9 months ago
676
Last month
677
 static void
678
 Coroutine_NS(RunNext)(
679
 void
680
 ){
9 months ago
681
 // arrive here with mutex unlocked
6 months ago
682
 _Cor_Mutex_Lock(&g_c->waiting_mutex);
683
 _Cor_Mutex_Lock(&g_c->mutex);
684
 Coroutine *next = List_Link_Container(Coroutine, link, List_GetHead(&g_c->runable));
Last month
685
 MyAssert(next->state == Coroutine_Running);
9 months ago
686
 longjmp(next->buf, Chunk_Enter);
4 months ago
687
 MyAssert(false);
9 months ago
688
 }
689
690
1 weeks ago
691
 /// @brief Ensures there's a spare Coroutine for cors
692
 /// @param cors The Coroutines to ensure a spare for
693
 /// @return true there is a spare; false there isn't
694
 static inline bool
695
 EnsureSpare
696
 (
697
 Coroutines *cors
698
 ){
699
 if (cors->spare){
700
 return true;
701
 }
702
 cors->spare = malloc(sizeof(*cors->spare));
703
 return cors->spare != NULL;
704
 }
705
706
Last month
707
 static Coroutine_Err
708
 Coroutines_ctor(
1 weeks ago
709
 Coroutines *cors,
710
 size_t min_size,
711
 size_t min_headroom
Last month
712
 ){
6 months ago
713
 cors->state = Coroutines_Starting;
4 months ago
714
 if (_Cor_Mutex_ctor(&cors->mutex)){
1 weeks ago
715
 goto error;
4 months ago
716
 }
6 months ago
717
 cors->primary = NULL;
5 months ago
718
 cors->stack_limit = g_stack_limit;
9 months ago
4
719
6 months ago
720
 List_Init(&cors->all);
6 months ago
721
 List_Init(&cors->free);
722
 List_Init(&cors->inactive);
723
 List_Init(&cors->runable);
724
 List_Init(&cors->waiting);
4 months ago
725
 if (_Cor_Mutex_ctor(&cors->waiting_mutex)){
1 weeks ago
726
 goto error1;
4 months ago
727
 }
728
 if (_Cor_Mutex_Lock(&cors->waiting_mutex)){
1 weeks ago
729
 goto error2;
4 months ago
730
 }
9 months ago
4
731
6 months ago
732
 cors->report.coroutines_created = 0;
733
 cors->report.coroutines_pool_size = 0;
734
 cors->report.largest_stack = 0;
1 weeks ago
735
 cors->spare = NULL;
736
 cors->sequence = 0;
6 months ago
737
1 weeks ago
738
 Coroutine *cor = malloc(sizeof(*cor));
739
 cor->min_size = min_size;
740
 cor->min_headroom = min_headroom;
741
 cor->coroutines = cors;
742
 cor->sequence = cors->sequence++;
743
 cor->state = Coroutine_Running;
744
 cor->base = (unsigned char *)&cor;
745
 if (cors->stack_limit){
746
 cor->limit = cors->stack_limit;
747
 cor->guard = StackPointerAdd(cor->limit, -GUARD_PATTERN_SIZE);
748
 Apply_Guard(cor->guard);
749
 } else {
750
 cor->limit = cor->guard = NULL;
9 months ago
4
751
 }
1 weeks ago
752
 cors->root = cor;
753
 List_AddHead(&cors->all, &cor->all_link);
754
 List_AddTail(&cors->runable, &cor->link);
755
 cors->tip = cors->active = cor;
9 months ago
756
1 weeks ago
757
 cors->report.coroutines_pool_size += 1;
6 months ago
758
759
 cors->state = Coroutines_Started;
4 months ago
760
 return Coroutine_OK;
1 weeks ago
761
 error2:
762
 _Cor_Mutex_dtor(&cors->waiting_mutex);
763
 error1:
764
 _Cor_Mutex_dtor(&cors->mutex);
765
 error:
766
 return Coroutine_Err_CouldNotInitialiseSystem;
4 months ago
767
 }
6 months ago
768
Last month
769
 static void
770
 Coroutines_dtor(
771
 Coroutines *cors
772
 ){
4 months ago
773
 _Cor_Mutex_Lock(&cors->mutex);
774
 cors->state = Coroutines_Stopping;
775
4 months ago
776
 MyAssert(List_IsEmpty(&cors->inactive));
1 weeks ago
777
778
 // free the spare
779
 if (cors->spare){
780
 free(cors->spare);
781
 }
782
783
 // free all non-spares
784
 List_Link *link;
785
 List_Link *next;
786
 for (link = List_Begin(&cors->all); Link_NextIsLink(link); link = next){
787
 next = Link_Next(link);
788
 Coroutine *cor = List_Link_Container(Coroutine, all_link, link) ;
789
 free(cor);
790
 }
791
4 months ago
792
 _Cor_Mutex_Unlock(&cors->waiting_mutex);
793
 _Cor_Mutex_dtor(&cors->waiting_mutex);
794
4 months ago
795
 MyAssert(cors->state == Coroutines_Stopping);
4 months ago
796
 _Cor_Mutex_Unlock(&cors->mutex);
797
 _Cor_Mutex_dtor(&cors->mutex);
9 months ago
4
798
 }
799
9 months ago
800
Last month
801
 Coroutine_Err
802
 Coroutine_NS(RunSystem)(
1 weeks ago
803
 size_t min_size,
804
 size_t min_headroom,
Last month
805
 Coroutine_SystemStart start,
806
 void *value
807
 ){
4 months ago
808
 CHECK_SYSTEM_NOT_RUNNING
809
810
 Coroutines cors;
1 weeks ago
811
 Coroutine_Err err = Coroutines_ctor(&cors, min_size, min_headroom);
4 months ago
812
 if (err){
813
 return err;
814
 }
815
 g_c = &cors;
1 weeks ago
816
 err = start(value, cors.root);
4 months ago
817
 g_c = NULL;
818
 Coroutines_dtor(&cors);
819
 return err;
820
 }
821
822
Last month
823
 void
824
 Coroutine_NS(SetStackLimit)(
825
 void *limit
826
 ){
4 months ago
827
 MyAssert(!limit || !g_c || !(g_c->state == Coroutines_Started || g_c->state == Coroutines_Active) || (unsigned char *)limit < (unsigned char *)g_c->tip || !g_c->tip);
5 months ago
828
 g_stack_limit = limit;
829
 if (g_c){
830
 g_c->stack_limit = limit;
1 weeks ago
831
 if (g_c->tip){
832
 g_c->tip->limit = limit;
833
 g_c->tip->guard = StackPointerAdd(limit, -GUARD_PATTERN_SIZE);
834
 Apply_Guard(g_c->tip->guard);
835
 }
5 months ago
836
 }
7 months ago
837
 }
838
839
4 months ago
840
 #if COROUTINE_RECORD_LOWEST_HEADROOM
Last month
841
 static size_t
842
 Coroutine_NS(UpdateMinimumHeadroom)(
843
 List_Head *list,
844
 size_t headroom
845
 ){
4 months ago
846
 for (List_Link *link = List_Begin(list); Link_NextIsLink(link); link = Link_Next(link)){
9 months ago
847
 Coroutine *cor = List_Link_Container(Coroutine, link, link);
7 months ago
848
 if (cor->guard){
1 weeks ago
849
 size_t chunk_size = Coroutine_Size(cor);
850
 for (size_t i = 0; i <= chunk_size-GUARD_PATTERN_SIZE; i += GUARD_PATTERN_SIZE){
851
 if (!Guard_Pattern_OK(StackPointerAdd(cor->guard, -i))){
4 months ago
852
 headroom = i < headroom ? i : headroom;
7 months ago
853
 break;
854
 }
9 months ago
855
 }
856
 }
857
 }
4 months ago
858
 return headroom;
859
 }
860
 #endif
861
862
Last month
863
 Coroutine_Report
864
 Coroutine_NS(GetReport)(
865
 void
866
 ){
4 months ago
867
 if (g_c){
868
 size_t headroom;
4 months ago
869
 #if COROUTINE_RECORD_LOWEST_HEADROOM
4 months ago
870
 _Cor_Mutex_Lock(&g_c->mutex);
871
 headroom = g_c->report.lowest_headroom;
Last month
872
 headroom = Coroutine_NS(UpdateMinimumHeadroom)(&g_c->inactive, headroom);
873
 headroom = Coroutine_NS(UpdateMinimumHeadroom)(&g_c->runable, headroom);
874
 headroom = Coroutine_NS(UpdateMinimumHeadroom)(&g_c->waiting, headroom);
4 months ago
875
 _Cor_Mutex_Unlock(&g_c->mutex);
7 months ago
876
 #else
4 months ago
877
 headroom = 0;
7 months ago
878
 #endif
4 months ago
879
 g_c->report.lowest_headroom = headroom;
9 months ago
880
4 months ago
881
 return g_c->report;
882
 } else {
883
 Coroutine_Report ret = {0, 0, 0, 0};
884
 return ret;
885
 }
4 months ago
886
 }
887
888
4 months ago
889
 #ifndef NDEBUG
Last month
890
 static void
891
 Coroutine_NS(ReportNonEmptyList)(
4 months ago
892
 List_Head const *head,
893
 char const *tag
894
 ){
895
 List_Link *link;
896
 for (link = List_Begin(head); Link_NextIsLink(link); link = Link_Next(link)){
897
 Coroutine *cor = List_Link_Container(Coroutine, link, link);
898
 printf("%s: %p %p %p\n", tag, cor, cor->start, cor->entry_param);
899
 }
900
 }
901
 #endif
902
Last month
903
 Coroutine_Err
904
 Coroutine_NS(Run_Coroutine)(
9 months ago
905
 Coroutine *cor,
9 months ago
906
 void *value
907
 ){
4 months ago
908
 CHECK_SYSTEM_RUNNING
909
 CHECK_COROUTINE_THREAD
910
 CHECK_NO_COROUTINE_RUNNING
911
9 months ago
912
 Coroutines *cors = cor->coroutines;
8 months ago
913
 _Cor_Mutex_Lock(&cors->mutex);
9 months ago
914
 cors->state = Coroutines_Active;
915
 cors->primary = cor;
9 months ago
916
Last month
917
 Coroutine_NS(Continue_)(cors, cor, value, true);
9 months ago
4
918
9 months ago
919
 if (!setjmp(cors->controller)){
Last month
920
 ready_jmp_buf(cors->controller);
8 months ago
921
 _Cor_Mutex_Unlock(&cors->mutex);
9 months ago
922
9 months ago
4
923
 // start the first coroutine
Last month
924
 Coroutine_NS(RunNext)();
9 months ago
4
925
 }
9 months ago
926
 // arrive here with mutex locked
4 months ago
927
 if (!List_IsEmpty(&cors->runable) || !List_IsEmpty(&cors->waiting)){
4 months ago
928
 #ifndef NDEBUG
Last month
929
 Coroutine_NS(ReportNonEmptyList)(&cors->runable, "runable");
930
 Coroutine_NS(ReportNonEmptyList)(&cors->waiting, "waiting");
4 months ago
931
 #endif
932
 return Coroutine_Err_ExitWithRunningCoroutines;
4 months ago
933
 }
4 months ago
934
 MyAssert(cors->state == Coroutines_Active);
9 months ago
935
 cors->state = Coroutines_Started;
8 months ago
936
 _Cor_Mutex_Unlock(&cors->mutex);
4 months ago
937
938
 return Coroutine_OK;
9 months ago
939
 }
940
941
Last month
942
 struct Coroutine_Run_Params {
4 months ago
943
 Coroutine_Start start;
944
 void *value;
945
 void **result;
946
 };
947
Last month
948
 static Coroutine_Err
949
 Coroutine_NS(Run_Starter)(
1 weeks ago
950
 void *_params,
951
 Coroutine *root
Last month
952
 ){
1 weeks ago
953
 (void)root;
Last month
954
 struct Coroutine_Run_Params *params = (struct Coroutine_Run_Params *)_params;
4 months ago
955
1 weeks ago
956
 void *res = params->start(params->value);
957
 if (params->result){
958
 *params->result = res;
4 months ago
959
 }
1 weeks ago
960
961
 return Coroutine_OK;
4 months ago
962
 }
963
964
Last month
965
 Coroutine_Err Coroutine_NS(Run)(
1 weeks ago
966
 size_t min_size,
967
 size_t min_headroom,
9 months ago
968
 Coroutine_Start start,
6 months ago
969
 void *value,
970
 void **result
9 months ago
971
 ){
4 months ago
972
 if (!g_c){
1 weeks ago
973
 struct Coroutine_Run_Params params = {start, value, result};
974
 return Coroutine_NS(RunSystem)(min_size, min_headroom, Coroutine_NS(Run_Starter), &params);
4 months ago
975
 }
976
977
 // We are in an active coroutine, so call start() directly
978
 CHECK_STACK_OVERRUN
979
 void *res = start(value);
6 months ago
980
 if (result){
4 months ago
981
 *result = res;
6 months ago
982
 }
4 months ago
983
6 months ago
984
 // no failures, so...
4 months ago
985
 return Coroutine_OK;
9 months ago
4
986
 }
987
988
Last month
989
 static Coroutine *Coroutine_NS(New_Lock_Assumed)(
1 weeks ago
990
 size_t min_size,
991
 size_t min_headroom,
6 months ago
992
 Coroutine_Start start
993
 ){
994
 List_Link *link;
995
1 weeks ago
996
 if (!EnsureSpare(g_c)){
997
 return NULL;
6 months ago
998
 }
999
6 months ago
1000
 Coroutine *cor = NULL;
6 months ago
1001
 for (link = List_Begin(&g_c->free); Link_NextIsLink(link); link = Link_Next(link)){
6 months ago
1002
 Coroutine *candidate = List_Link_Container(Coroutine, link, link);
4 months ago
1003
 MyAssert(candidate->coroutines == g_c);
1 weeks ago
1004
 MyAssert(candidate->state == Coroutine_Free);
1005
 if (candidate->limit){
1006
 size_t candidate_size = Coroutine_Size(candidate);
1007
 if (candidate_size < min_size){
1008
 continue;
7 months ago
1009
 }
1 weeks ago
1010
 }
1011
 // candidate has no limit, or is big enough
6 months ago
1012
1 weeks ago
1013
 // check if it's 'better' (lower in the C stack) than cor
1014
 if (!cor || StackPointerDiff(candidate->base, cor->base) < 0){
6 months ago
1015
 // chunk big enough, and a better choice than cor
1016
 cor = candidate;
1017
 }
1018
 }
1019
1 weeks ago
1020
 if (!cor){
1021
 return NULL;
1022
 }
1023
1024
 // use the whole block - the tail will be freed in (New) or (Yield)
1025
 cor->min_size = min_size;
1026
 cor->min_headroom = min_headroom;
1027
 cor->state = Coroutine_Idle;
1028
 cor->start = start;
1029
 cor->value = NULL;
1030
 Link_Remove(&cor->link);
1031
 List_AddHead(&g_c->inactive, &cor->link);
1032
1033
 // trim down to an appropriate size
1034
 size_t trimmable_size = TrimmableSize(cor);
1035
 if (trimmable_size){
1036
 // split block into two
1037
 if (EnsureSpare(g_c)){
1038
 g_c->size_to_retain = trimmable_size;
6 months ago
1039
 if (!setjmp(g_c->chunk_allocated)){
Last month
1040
 ready_jmp_buf(g_c->chunk_allocated);
6 months ago
1041
 longjmp(cor->buf, Chunk_Split);
6 months ago
1042
 }
1043
 }
9 months ago
4
1044
 }
1045
1 weeks ago
1046
 g_c->report.coroutines_created += 1;
6 months ago
1047
1 weeks ago
1048
 return cor;
1049
 }
1050
1051
1052
 static void
1053
 TrimActiveIfPossible
1054
 (
1055
 void
1056
 ){
1057
 MyAssert(g_c);
1058
1059
 // If we're the active coroutine, free the tail
1060
 Coroutine *active = g_c->active;
1061
 MyAssert(active);
1062
1063
 active->stack_top = (unsigned char *)StackTopNow();
1064
 ptrdiff_t timmablesize = TrimmableSize(active);
1065
 if (!timmablesize){
1066
 return;
6 months ago
1067
 }
1 weeks ago
1068
1069
 if (!EnsureSpare(g_c)){
1070
 return;
6 months ago
1071
 }
1072
1 weeks ago
1073
 // enough space for a second coroutine so free the unused stack space from active
1074
 if (!setjmp(g_c->chunk_allocated)){
1075
 ready_jmp_buf(g_c->chunk_allocated);
1076
 ReserveStackSpace(g_c, active, timmablesize - StackPointerDiff(active->stack_top, active->base), active->limit);
1077
 MyAssert(false);
1078
 }
1079
 }
1080
1081
1082
 static void
1083
 EnlargeActiveIfPossible(
1084
 void
1085
 ){
1086
 MyAssert(g_c);
1087
 Coroutine *active = g_c->active;
1088
 MyAssert(active);
1089
1090
 List_Link *next = Link_Next(&active->all_link);
1091
 if (!Link_NextIsLink(next)){
1092
 return;
1093
 }
1094
 Coroutine *cor = List_Link_Container(Coroutine, all_link, next);
1095
 if (cor->state != Coroutine_Free){
1096
 return;
1097
 }
1098
1099
 // Following block is free, merge with this
1100
 active->limit = cor->limit;
1101
 active->guard = cor->guard;
6 months ago
1102
 Link_Remove(&cor->link);
1 weeks ago
1103
 Link_Remove(&cor->all_link);
1104
 if (g_c->tip == cor){
1105
 g_c->tip = active;
1106
 }
1107
 if (g_c->spare){
1108
 free(cor);
1109
 } else {
1110
 g_c->spare = cor;
1111
 }
6 months ago
1112
 }
9 months ago
1113
6 months ago
1114
Last month
1115
 Coroutine *
1116
 Coroutine_NS(New)(
1 weeks ago
1117
 size_t min_size,
1118
 size_t min_headroom,
6 months ago
1119
 Coroutine_Start start
1120
 ){
4 months ago
1121
 MyAssert(g_c);
1122
 MyAssert((g_c->state == Coroutines_Started && List_IsEmpty(&g_c->inactive)) || g_c->state == Coroutines_Active);
Last month
1123
 MyAssert(!Coroutine_NS(StackHasOverrun)());
6 months ago
1124
1 weeks ago
1125
 // Make the paramaters make sense
1126
 if (min_size < min_headroom){
1127
 min_size = min_headroom;
1128
 }
1129
6 months ago
1130
 _Cor_Mutex_Lock(&g_c->mutex);
1131
1 weeks ago
1132
 TrimActiveIfPossible();
6 months ago
1133
1 weeks ago
1134
 Coroutine *cor = Coroutine_NS(New_Lock_Assumed)(min_size, min_headroom, start);
1135
1136
 if (cor && Coroutine_Size(cor) > g_c->report.largest_stack){
1137
 g_c->report.largest_stack = Coroutine_Size(cor);
6 months ago
1138
 }
1139
1 weeks ago
1140
 EnlargeActiveIfPossible();
1141
6 months ago
1142
 _Cor_Mutex_Unlock(&g_c->mutex);
1143
9 months ago
4
1144
 return cor;
1145
 }
1146
9 months ago
1147
Last month
1148
 void
1149
 Coroutine_NS(Delete)(
9 months ago
1150
 Coroutine *cor
1151
 ){
Last month
1152
 MyAssert(!Coroutine_NS(StackHasOverrun)());
6 months ago
1153
 if (cor){
1154
 Coroutines *cors = cor->coroutines;
1155
 _Cor_Mutex_Lock(&cors->mutex);
Last month
1156
 MyAssert(cor->state == Coroutine_Idle || cor->state == Coroutine_Complete);
4 months ago
1157
1158
 #if COROUTINE_RECORD_LOWEST_HEADROOM
1159
 if (cor->guard){
1 weeks ago
1160
 unsigned char *guard = cor->guard;
1161
 ptrdiff_t chunk_size = Coroutine_Size(cor);
1162
 ptrdiff_t myheadroom;
1163
 for (myheadroom = 0; myheadroom <= chunk_size-(ptrdiff_t)GUARD_PATTERN_SIZE; myheadroom += GUARD_PATTERN_SIZE){
1164
 if (!Guard_Pattern_OK(StackPointerAdd(guard, -myheadroom))){
4 months ago
1165
 break;
1166
 }
1167
 }
1 weeks ago
1168
 if (myheadroom < (ptrdiff_t)g_c->report.lowest_headroom || g_c->report.lowest_headroom == 0){
4 months ago
1169
 g_c->report.lowest_headroom = myheadroom;
1170
 }
1171
 }
1172
 #endif
1173
Last month
1174
 cor->state = Coroutine_Free;
6 months ago
1175
 Link_Remove(&cor->link);
1176
6 months ago
1177
 // insert into free list
1178
 List_AddHead(&cors->free, &cor->link);
6 months ago
1179
1180
 // Check for merge with following Coroutine
6 months ago
1181
 List_Link *link = Link_Next(&cor->all_link);
6 months ago
1182
 if (Link_NextIsLink(link)){
6 months ago
1183
 Coroutine *listcor = List_Link_Container(Coroutine, all_link, link);
Last month
1184
 if (listcor->state == Coroutine_Free){
6 months ago
1185
 // merge
1186
 cor->limit = listcor->limit;
1187
 cor->guard = listcor->guard;
6 months ago
1188
 Link_Remove(&listcor->all_link);
6 months ago
1189
 Link_Remove(&listcor->link);
6 months ago
1190
 if (g_c->tip == listcor){
1191
 g_c->tip = cor;
1192
 }
1 weeks ago
1193
1194
 // free up the now unused struct
1195
 if (g_c->spare){
1196
 free(listcor);
1197
 } else {
1198
 g_c->spare = listcor;
1199
 }
6 months ago
1200
 }
1201
 }
1202
1203
 // check for merge with prev coroutine
6 months ago
1204
 link = Link_Prev(&cor->all_link);
6 months ago
1205
 if (Link_PrevIsLink(link)){
6 months ago
1206
 Coroutine *listcor = List_Link_Container(Coroutine, all_link, link);
Last month
1207
 if (listcor->state == Coroutine_Free){
6 months ago
1208
 // merge
1209
 listcor->limit = cor->limit;
1210
 listcor->guard = cor->guard;
6 months ago
1211
 Link_Remove(&cor->all_link);
6 months ago
1212
 Link_Remove(&cor->link);
6 months ago
1213
 if (g_c->tip == cor){
1214
 g_c->tip = listcor;
1215
 }
1 weeks ago
1216
1217
 // free up the now unused struct
1218
 if (g_c->spare){
1219
 free(cor);
1220
 } else {
1221
 g_c->spare = cor;
1222
 }
6 months ago
1223
 }
1224
 }
1 weeks ago
1225
1226
 EnlargeActiveIfPossible();
6 months ago
1227
6 months ago
1228
 _Cor_Mutex_Unlock(&cors->mutex);
1229
 }
9 months ago
4
1230
 }
1231
9 months ago
1232
Last month
1233
 // Coroutine_NS(Continue), assuming the mutex is claimed
4 months ago
1234
 // return false for success, true for something went wrong
Last month
1235
 static Coroutine_Err
1236
 Coroutine_NS(Continue_)(
4 months ago
1237
 Coroutines *cors,
9 months ago
1238
 Coroutine *cor,
1239
 void *value,
1240
 bool early
1241
 ){
Last month
1242
 if (cor->state == Coroutine_Running){
4 months ago
1243
 // already running
4 months ago
1244
 return Coroutine_OK;
4 months ago
1245
 }
Last month
1246
 if (cor->state != Coroutine_Idle && cor->state != Coroutine_Waiting){
4 months ago
1247
 return Coroutine_Err_WrongState;
1248
 }
9 months ago
4
1249
 cor->entry_param = value;
Last month
1250
 cor->state = Coroutine_Running;
6 months ago
1251
 Link_Remove(&cor->link);
9 months ago
4
1252
 if ( early ) {
9 months ago
1253
 List_AddHead(&cors->runable, &cor->link);
9 months ago
4
1254
 } else {
9 months ago
1255
 List_AddTail(&cors->runable, &cor->link);
9 months ago
4
1256
 }
8 months ago
1257
 _Cor_Mutex_Unlock(&cors->waiting_mutex);
4 months ago
1258
 return Coroutine_OK;
8 months ago
1259
 }
1260
1261
Last month
1262
 Coroutine_Err
1263
 Coroutine_NS(Continue)(
8 months ago
1264
 Coroutine *cor,
1265
 void *value,
1266
 bool early
1267
 ){
Last month
1268
 MyAssert(!Coroutine_NS(StackHasOverrun)());
8 months ago
1269
 Coroutines *cors = cor->coroutines;
8 months ago
1270
 _Cor_Mutex_Lock(&cors->mutex);
Last month
1271
 Coroutine_Err err = Coroutine_NS(Continue_)(cors, cor, value, early);
8 months ago
1272
 _Cor_Mutex_Unlock(&cors->mutex);
4 months ago
1273
 return err;
9 months ago
4
1274
 }
1275
9 months ago
1276
Last month
1277
 void *
1278
 Coroutine_NS(Yield)(
9 months ago
1279
 void *value,
Last month
1280
 Coroutine_YieldCallback on_yield,
9 months ago
1281
 void *yield_me
9 months ago
1282
 ){
4 months ago
1283
 MyAssert(g_c);
6 months ago
1284
 Coroutine *me = g_c->active;
4 months ago
1285
 MyAssert(me);
Last month
1286
 MyAssert(!Coroutine_NS(StackHasOverrun)());
9 months ago
1287
6 months ago
1288
 _Cor_Mutex_Lock(&g_c->mutex);
9 months ago
1289
 Coroutines *cors = me->coroutines;
Last month
1290
 MyAssert(me && me->state == Coroutine_Running && cors == g_c);
Last month
1291
 me->stack_top = (unsigned char *)StackTopNow();
9 months ago
4
1292
 me->value = value;
Last month
1293
 me->state = Coroutine_Waiting;
9 months ago
1294
1 weeks ago
1295
 TrimActiveIfPossible();
1296
6 months ago
1297
 Link_Remove(&me->link);
8 months ago
1298
 if (!List_IsEmpty(&cors->runable)){
8 months ago
1299
 _Cor_Mutex_Unlock(&cors->waiting_mutex);
8 months ago
1300
 }
9 months ago
1301
 List_AddTail(&cors->waiting, &me->link);
9 months ago
1302
9 months ago
1303
 switch (setjmp(me->buf)){
1304
 case Chunk_Initial:
Last month
1305
 ready_jmp_buf(me->buf);
8 months ago
1306
 _Cor_Mutex_Unlock(&cors->mutex);
9 months ago
1307
 on_yield(yield_me);
Last month
1308
 Coroutine_NS(RunNext)();
4 months ago
1309
 MyAssert(false);
3 months ago
1310
 break;
9 months ago
1311
 case Chunk_Enter:
1312
 // arrive here with mutex locked
9 months ago
1313
 cors->active = me;
Last month
1314
 MyAssert(!Coroutine_NS(StackHasOverrun)());
9 months ago
1315
 // when we return here - we are running again
Last month
1316
 MyAssert(me->state == Coroutine_Running);
1 weeks ago
1317
 EnlargeActiveIfPossible();
9 months ago
1318
 void *res = me->entry_param;
8 months ago
1319
 _Cor_Mutex_Unlock(&cors->mutex);
9 months ago
1320
 return res;
9 months ago
4
1321
 }
3 months ago
1322
 MyAssert(false);
9 months ago
1323
 return NULL;
9 months ago
4
1324
 }
1325
9 months ago
1326
Last month
1327
 void *
1328
 Coroutine_NS(GetValue)(
9 months ago
1329
 Coroutine *cor
1330
 ){
9 months ago
4
1331
 return cor->value;
1332
 }
1333
9 months ago
1334
Last month
1335
 Coroutine *
1336
 Coroutine_NS(GetActive)(
1337
 void
1338
 ){
6 months ago
1339
 return g_c ? g_c->active : NULL;
9 months ago
4
1340
 }
1341
9 months ago
1342
1 weeks ago
1343
 ptrdiff_t
Last month
1344
 Coroutine_NS(GetStackHeadroom)(
1345
 void
1346
 ){
4 months ago
1347
 Coroutine *me = g_c ? g_c->active : NULL;
7 months ago
1348
 if (!me){
1349
 // no active coroutine
4 months ago
1350
 if (g_stack_limit){
1 weeks ago
1351
 return StackPointerDiff(g_stack_limit, (unsigned char *)StackTopNow());
7 months ago
1352
 } else {
1353
 // no information where the stack ends - return something
1 weeks ago
1354
 // The biggest ptrdiff_t possible
1355
 return PTRDIFF_MAX;
7 months ago
1356
 }
1 weeks ago
1357
 } else {
1358
 if (me->guard){
1359
 return StackPointerDiff(me->guard, (unsigned char *)StackTopNow());
1360
 } else {
1 weeks ago
1361
 // The biggest ptrdiff_t possible
1362
 return PTRDIFF_MAX;
1 weeks ago
1363
 }
7 months ago
1364
 }
9 months ago
1365
 }
1366
1367
Last month
1368
 void *
1369
 Coroutine_NS(GetStackHWM)(
1370
 void
1371
 ){
4 months ago
1372
 MyAssert(g_c);
1373
 MyAssert(g_c->state == Coroutines_Active);
Last month
1374
 MyAssert(!Coroutine_NS(StackHasOverrun)());
7 months ago
1375
 // Find where the guards end
1 weeks ago
1376
 unsigned char *guard = g_c->active->guard;
1377
 if (guard){
1378
 ptrdiff_t headroom = Coroutine_NS(GetStackHeadroom)();
1379
 for (ptrdiff_t i = 0; i <= headroom; i += GUARD_PATTERN_SIZE){
1380
 if (!Guard_Pattern_OK(StackPointerAdd(guard, -i))){
1381
 return StackPointerAdd(guard, i);
1382
 }
1383
 }
7 months ago
1384
 }
1385
 return guard;
1386
 }
8 months ago
1387
1388
Last month
1389
 void
1390
 Coroutine_NS(ClearStackForHWM)(
1391
 void
1392
 ){
4 months ago
1393
 MyAssert(g_c);
1394
 MyAssert(g_c->state == Coroutines_Active);
Last month
1395
 MyAssert(!Coroutine_NS(StackHasOverrun)());
1 weeks ago
1396
 unsigned char *guard = g_c->active->guard;
1397
 if (guard){
1398
 ptrdiff_t headroom = Coroutine_NS(GetStackHeadroom)();
1399
 for (ptrdiff_t i = 0; i <= headroom; i += GUARD_PATTERN_SIZE){
1400
 Apply_Guard(StackPointerAdd(guard, -i));
1401
 }
7 months ago
1402
 }
1403
 }
1404
1405
Last month
1406
 static bool
1407
 Coroutine_NS(CanStartCoroutine_Lock_Assumed)(
1 weeks ago
1408
 size_t min_size
6 months ago
1409
 ){
1410
 if (!g_c->stack_limit){
7 months ago
1411
 return true;
1412
 }
7 months ago
1413
1 weeks ago
1414
 // check free list
6 months ago
1415
 List_Link *link;
1416
 for (link = List_Begin(&g_c->free); Link_NextIsLink(link); link = Link_Next(link)){
1417
 Coroutine *cor = List_Link_Container(Coroutine, link, link);
1 weeks ago
1418
 size_t cor_size = Coroutine_Size(cor);
1419
 if (cor_size >= min_size){
6 months ago
1420
 return true;
1421
 }
1422
 }
1423
1 weeks ago
1424
 // Check if this coroutine has enough size
1425
 Coroutine *me = g_c->active;
1426
 me->stack_top = (unsigned char *)StackTopNow();
1427
 ptrdiff_t reduced_size = TrimmableSize(g_c->active);
1428
 if (reduced_size && StackPointerDiff(g_c->active->limit, g_c->active->base) - reduced_size > (ptrdiff_t)min_size){
1429
 return true;
1430
 }
1431
6 months ago
1432
 return false;
7 months ago
1433
 }
1434
1435
Last month
1436
 bool
1437
 Coroutine_NS(CanStartCoroutine)(
6 months ago
1438
 size_t size
1439
 ){
4 months ago
1440
 MyAssert(g_c);
1441
 MyAssert(g_c->state == Coroutines_Started || g_c->state == Coroutines_Active);
Last month
1442
 MyAssert(!Coroutine_NS(StackHasOverrun)());
4 months ago
1443
6 months ago
1444
 _Cor_Mutex_Lock(&g_c->mutex);
1445
Last month
1446
 bool result = Coroutine_NS(CanStartCoroutine_Lock_Assumed)(size);
6 months ago
1447
1448
 _Cor_Mutex_Unlock(&g_c->mutex);
1449
1450
 return result;
1451
 }
1452
Last month
1453
 void *
1454
 Coroutine_NS(GetCStackTop)(
1455
 void
1456
 ){
1 weeks ago
1457
 if (!g_c || g_c->tip == g_c->active){
1458
 return (void *)StackTopNow();
7 months ago
1459
 }
1 weeks ago
1460
1461
 return g_c->tip->stack_top;
9 months ago
1462
 }
1463
1464
Last month
1465
 // Inspired by cpython...
1466
 #ifdef __has_builtin
1467
 # define Coroutine__has_builtin(x) __has_builtin(x)
1468
 #else
1469
 # define Coroutine__has_builtin(x) 0
1470
 #endif
1471
1472
 #if !Coroutine__has_builtin(__builtin_frame_address) && !defined(__GNUC__) && !defined(_MSC_VER)
1473
 static uintptr_t return_pointer_as_int(char* p) {
1474
 return (uintptr_t)p;
7 months ago
1475
 }
Last month
1476
 #endif
7 months ago
1477
Last month
1478
 static inline uintptr_t
1479
 StackTopNow(void) {
1480
 #if Coroutine__has_builtin(__builtin_frame_address) || defined(__GNUC__)
1481
 return (uintptr_t)__builtin_frame_address(0);
1482
 #elif defined(_MSC_VER)
1483
 return (uintptr_t)_AddressOfReturnAddress();
1484
 #else
1485
 char here;
1486
 /* Avoid compiler warning about returning stack address */
1487
 return return_pointer_as_int(&here);
1488
 #endif
1489
 }
1490
 // ...inspired by cpython
7 months ago
1491
Last month
1492
Last month
1493
 struct Coroutine_ChainParam {
9 months ago
1494
 Coroutine_Start start;
1495
 void *value;
1496
 Coroutine *ret;
1497
 };
1498
1499
Last month
1500
 static void *
1501
 Coroutine_NS(ChainFn)(
9 months ago
1502
 void *param
1503
 ){
Last month
1504
 struct Coroutine_ChainParam *params = (struct Coroutine_ChainParam *)param;
Last month
1505
 return (void *)(uintptr_t)Coroutine_NS(Continue)(params->ret, params->start(params->value), true);
9 months ago
1506
 }
1507
1508
Last month
1509
 static void
1510
 Coroutine_NS(ChainYield)(
9 months ago
1511
 void *unused
1512
 ){
1513
 (void)unused;
1514
 }
1515
1516
Last month
1517
 Coroutine_Err
1518
 Coroutine_NS(Chain)(
1 weeks ago
1519
 size_t min_size,
1520
 size_t min_headroom,
9 months ago
1521
 Coroutine_Start start,
6 months ago
1522
 void *value,
1523
 void **result
9 months ago
1524
 ){
Last month
1525
 MyAssert(!Coroutine_NS(StackHasOverrun)());
1 weeks ago
1526
 Coroutine *cor = Coroutine_NS(New)(min_size, min_headroom, Coroutine_NS(ChainFn));
6 months ago
1527
 if (!cor){
1528
 // failed
4 months ago
1529
 return Coroutine_Err_NoStack;
6 months ago
1530
 }
Last month
1531
 struct Coroutine_ChainParam params = {
9 months ago
1532
 start,
1533
 value,
Last month
1534
 Coroutine_NS(GetActive)()
9 months ago
1535
 };
Last month
1536
 Coroutine_Err err = Coroutine_NS(Continue)(cor, &params, true);
4 months ago
1537
 if (err){
1538
 return err;
1539
 }
Last month
1540
 void *res = Coroutine_NS(Yield)(NULL, Coroutine_NS(ChainYield), NULL);
1541
 err = (Coroutine_Err)(uintptr_t)Coroutine_NS(GetValue)(cor);
1542
 Coroutine_NS(Delete)(cor);
4 months ago
1543
 if (!err && result){
6 months ago
1544
 *result = res;
1545
 }
4 months ago
1546
 // success! ...probably
1547
 return err;
9 months ago
1548
 }
1549
1550
Last month
1551
 bool
1552
 Coroutine_NS(IsRunning)(
9 months ago
1553
 Coroutine *cor
Last month
1554
 ){
9 months ago
1555
 int state = cor->state;
Last month
1556
 return state == Coroutine_Running || state == Coroutine_Waiting;
9 months ago
4
1557
 }
9 months ago
1558
1559
Last month
1560
 bool Coroutine_NS(IsComplete)(
7 months ago
1561
 Coroutine *cor
Last month
1562
 ){
7 months ago
1563
 int state = cor->state;
Last month
1564
 return state == Coroutine_Complete;
7 months ago
1565
 }
1566
1567
Last month
1568
 bool
1569
 Coroutine_NS(IsStarted)(
1570
 void
1571
 ){
6 months ago
1572
 return g_c && (g_c->state == Coroutines_Active || g_c->state == Coroutines_Started);
9 months ago
1573
 }
6 months ago
1574
Last month
1575
 void
1576
 Coroutine_NS(Dump_)(
1577
 void
1578
 ){
6 months ago
1579
 char *state_to_text[] = {
1580
 "Free",
1581
 "Idle",
1582
 "Running",
1583
 "Waiting",
1584
 "Complete"
1585
 };
1586
 unsigned idx = 0;
1587
 List_Link *link;
1588
 for (link = List_Begin(&g_c->all); Link_NextIsLink(link); link = Link_Next(link)){
1589
 Coroutine *cor = List_Link_Container(Coroutine, all_link, link);
1 weeks ago
1590
 printf("%d) %p (%s) %s\n", idx++, cor, state_to_text[cor->state], cor == g_c->tip ? " (TIP)" : "");
6 months ago
1591
 }
1592
 }
1593