0 branches 0 tags
108
Fix transfer of guard to child coroutines
jonroach 5 Aug 2026 11:52
1 contributor
1686 lines42.1 KB
Newer
Older
Text Raw Blame
  • Delete
-
+
commited
{line.log.rev}
 on
11 months ago
4
1
 #include "coroutine.h"
2
 #include <assert.h>
3
 #include <setjmp.h>
4
 #include <stdbool.h>
5
 #include <stddef.h>
8 months ago
6
 #include <stdio.h>
2 months ago
7
 #include <stdlib.h>
10 months ago
8
 #include "cor_platform.h"
11 months ago
4
9
2 months ago
10
 #include "coroutine_names_def.h"
11
9 months ago
12
 // see CPython again, this time from ctypes.h
13
 #if (defined (__SVR4) && defined (__sun)) || defined(COROUTINE_HAVE_ALLOCA_H)
14
 # include <alloca.h>
15
 #elif defined(MS_WIN32)
16
 # include <malloc.h>
17
 #endif
11 months ago
4
18
9 months ago
19
 /* If the system does not define alloca(), we have to hope for a compiler builtin. */
20
 #ifndef alloca
21
 # if defined __GNUC__ || (__clang_major__ >= 4)
22
 # define alloca __builtin_alloca
23
 # else
24
 # error "Could not define alloca() on your platform."
25
 # endif
26
 #endif
27
6 months ago
28
 typedef struct Coroutines Coroutines;
29
2 months ago
30
 static void Coroutine_RunNext(void);
31
 static Coroutine_Err Coroutine_Continue_(Coroutines *cors, Coroutine *cor, void *value, bool early);
3 months ago
32
 static uintptr_t StackTopNow(void);
2 months ago
33
 static void TrimActiveIfPossible(void);
34
 static void EnlargeActiveIfPossible(void);
11 months ago
35
6 months ago
36
 #ifndef NDEBUG
37
 // In debug builds, use the built-in assert
38
 #define MyAssert assert
39
 #else
40
 #if 1
41
 // In non-debug builds, normally use this - all the asserts are disabled
42
 #define MyAssert(cond)
43
 #else
44
 // In non-debug builds with stack problems, you can use this.
45
 // This activates all the asserts, and gives a line to put a
46
 // breakpoint in your debugger.
47
 static void _MyAssert(bool cond, char const *msg)
48
 {
49
 if (!cond){
50
 fputs("Assertion failed: ", stdout);
51
 fputs(msg, stdout);
52
 fputs("\n", stdout);
53
 }
54
 }
55
 #define MyAssert(cond) _MyAssert(cond, #cond)
56
 #endif
57
 #endif
58
2 months ago
59
 static inline ptrdiff_t
60
 StackPointerDiff
61
 (
62
 unsigned char *a,
63
 unsigned char *b
64
 ){
65
 #if COROUTINE_STACK_GROWS_UP
66
 return a - b;
67
 #else
68
 return b - a;
69
 #endif
70
 }
71
72
 static inline unsigned char *
73
 StackPointerAdd
74
 (
75
 unsigned char *a,
76
 ptrdiff_t b
77
 ){
78
 #if COROUTINE_STACK_GROWS_UP
79
 return a + b;
80
 #else
81
 return a - b;
82
 #endif
83
 }
84
85
 // Unused...
86
 // static inline unsigned char *
87
 // StackBaseEnd
88
 // (
89
 // unsigned char *memlo,
90
 // unsigned char *memhi
91
 // ){
92
 // #if COROUTINE_STACK_GROWS_UP
93
 // (void)memhi;
94
 // return memlo;
95
 // #else
96
 // (void)memlo;
97
 // return memhi-1;
98
 // #endif
99
 // }
100
 // ...unused
101
102
 static inline unsigned char *
103
 StackLimitEnd
104
 (
105
 unsigned char *memlo,
106
 unsigned char *memhi
107
 ){
108
 #if COROUTINE_STACK_GROWS_UP
109
 (void)memlo;
110
 return memhi;
111
 #else
112
 (void)memhi;
113
 return memlo-1;
114
 #endif
115
 }
116
6 months ago
117
 #define CHECK_SYSTEM_RUNNING \
118
 if (!g_c){ \
119
 return Coroutine_Err_SystemNotRunning; \
120
 }
121
 #define CHECK_SYSTEM_NOT_RUNNING \
122
 if (g_c){ \
123
 return Coroutine_Err_SystemRunning; \
124
 }
125
 #define CHECK_COROUTINE_THREAD \
126
 if (cor->coroutines != g_c){ \
127
 return Coroutine_Err_CoroutineFromWrongThread; \
128
 }
129
 #define CHECK_NO_COROUTINE_RUNNING \
130
 if (g_c->state != Coroutines_Started){ \
131
 return Coroutine_Err_ACoroutineIsAlreadyRunning; \
132
 }
133
 #define CHECK_STACK_OVERRUN \
134
 { \
2 months ago
135
 Coroutine_Err err = Coroutine_StackHasOverrun(); \
6 months ago
136
 if (err){ \
137
 return err; \
138
 } \
139
 } while (0);
140
3 months ago
141
142
 static inline void ready_jmp_buf(jmp_buf buf) {
143
 #if defined(_M_X64) || defined(_M_ARM64)
144
 // Win64:
145
 // Set Frame to 0 on Windows 64 bit to prevent C++ stack unwinding in longjmp().
146
 // Win32:
147
 // Doesn't do this, so only needed on the 2 64 bit Windows versions
148
 ((_JUMP_BUFFER*)buf)->Frame = 0;
3 months ago
149
 #else
150
 (void)buf;
3 months ago
151
 #endif
152
 }
153
11 months ago
4
154
 ///////////////////////////////////////////////////////////////////////////////
155
 // 2-way linked lists...
156
 //
11 months ago
157
 // Brought inline here to avoid namespace polution
11 months ago
4
158
 ///////////////////////////////////////////////////////////////////////////////
159
160
 typedef struct List_Link List_Link;
161
 struct List_Link {
162
 List_Link *next;
163
 List_Link *prev;
164
 };
165
166
 typedef struct List_Head List_Head;
167
 struct List_Head {
168
 union {
169
 struct {
170
 List_Link link;
171
 List_Link *filler;
172
 } fwd;
173
 struct {
174
 List_Link *filler;
175
 List_Link link;
176
 } back;
177
 };
178
 };
179
11 months ago
180
181
 static inline bool List_IsEmpty(
182
 const List_Head *list
183
 ){
11 months ago
4
184
 return list->fwd.link.next == &list->back.link;
185
 }
186
11 months ago
187
188
 static inline List_Link *List_GetHead(
189
 const List_Head *list
190
 ){
11 months ago
4
191
 return List_IsEmpty(list) ? NULL : list->fwd.link.next;
192
 }
11 months ago
193
194
8 months ago
195
 static inline List_Link *List_Begin(
196
 const List_Head *list
197
 ){
198
 return list->fwd.link.next;
199
 }
200
201
202
 static inline bool Link_NextIsLink(
203
 const List_Link *link
204
 ){
205
 return link->next != NULL;
206
 }
207
208
209
 static inline List_Link *Link_Next(
210
 List_Link *link
211
 ){
212
 return link->next;
213
 }
214
215
216
 static inline bool Link_PrevIsLink(
217
 const List_Link *link
218
 ){
219
 return link->prev != NULL;
220
 }
221
222
223
 static inline List_Link *Link_Prev(
224
 List_Link *link
225
 ){
226
 return link->prev;
227
 }
228
2 months ago
229
 // Unused...
230
 // static inline List_Link *List_GetTail(
231
 // const List_Head *list
232
 // ){
233
 // return List_IsEmpty(list) ? NULL : list->back.link.prev;
234
 // }
235
 // ...unused
11 months ago
236
237
11 months ago
4
238
 #define OFFSETOF(Container, Field) ((char *)&((Container *)4)->Field - (char *)(Container *)4)
239
 #define List_Link_Container(Container, Link, link) ((Container *)((char *)(link) - OFFSETOF(Container, Link)))
240
11 months ago
241
3 months ago
242
 static inline void
243
 List_Init(
11 months ago
244
 List_Head *list
245
 ){
11 months ago
4
246
 list->fwd.link.next = &list->back.link;
247
 list->fwd.link.prev = NULL;
248
 list->back.link.prev = &list->fwd.link;
249
 }
250
11 months ago
251
3 months ago
252
 static inline void
253
 Link_AddAfter(
8 months ago
254
 List_Link *link,
255
 List_Link *after
256
 ){
257
 link->next = after->next;
258
 link->prev = after;
259
 after->next->prev = link;
260
 after->next = link;
261
 }
262
263
3 months ago
264
 static inline void
265
 List_AddHead(
11 months ago
266
 List_Head *list,
267
 List_Link *link
268
 ){
8 months ago
269
 Link_AddAfter(link, &list->fwd.link);
11 months ago
4
270
 }
271
11 months ago
272
3 months ago
273
 static inline void
274
 Link_AddBefore(
8 months ago
275
 List_Link *link,
276
 List_Link *before
277
 ){
278
 link->prev = before->prev;
279
 link->next = before;
280
 before->prev->next = link;
281
 before->prev = link;
282
 }
283
284
3 months ago
285
 static inline void
286
 List_AddTail(
11 months ago
287
 List_Head *list,
288
 List_Link *link
289
 ){
8 months ago
290
 Link_AddBefore(link, &list->back.link);
11 months ago
4
291
 }
292
11 months ago
293
3 months ago
294
 static inline void
295
 Link_Remove(
11 months ago
296
 List_Link *link
297
 ){
11 months ago
4
298
 link->prev->next = link->next;
299
 link->next->prev = link->prev;
300
 }
301
302
 ///////////////////////////////////////////////////////////////////////////////
303
 // ...2-way linked lists
304
 ///////////////////////////////////////////////////////////////////////////////
305
306
 enum {
307
 Coroutines_Started,
308
 Coroutines_Stopping
309
 };
310
311
 enum {
312
 Chunk_Initial,
8 months ago
313
 Chunk_Split,
11 months ago
4
314
 Chunk_Enter
315
 };
316
11 months ago
317
 typedef enum Coroutine_State {
3 months ago
318
 Coroutine_Free,
319
 Coroutine_Idle,
320
 Coroutine_Running,
321
 Coroutine_Waiting,
322
 Coroutine_Complete
11 months ago
323
 } Coroutine_State;
11 months ago
4
324
325
 enum {
326
 Coroutines_Init,
327
 Coroutines_AllocatedChunk,
328
 Coroutines_CoroutineComplete,
329
 };
330
331
 struct Coroutine {
2 months ago
332
 size_t min_size;
333
 size_t min_headroom;
9 months ago
334
 Coroutines *coroutines; // so can work with it off-thread
335
 List_Link link; // for whichever list it's on
8 months ago
336
 List_Link all_link; // list of all Coroutines
9 months ago
337
 jmp_buf buf; // how to get back to it
2 months ago
338
 unsigned char *base; // where the base of this Coroutine's stack is (= previous block's limit)
339
 unsigned char *limit; // where the limit of this Coroutine's stack is (= next block's base)
9 months ago
340
 unsigned char *guard; // where the stack overrun guard is
341
 Coroutine_Start start; // entry point
342
 void *entry_param; // to pass to start
343
 void *value; // yielded/returned
344
 unsigned char *stack_top; // recorded at yield
Last month
345
 struct Coroutine *chain_root; // The root (entry) of a series of chained coroutines
346
 struct Coroutine *chain_tip; // The tip (yielder) of a series of chained coroutines
11 months ago
347
 Coroutine_State state;
2 months ago
348
349
 int sequence;
11 months ago
4
350
 };
351
352
 struct Coroutines {
10 months ago
353
 _Cor_Mutex mutex;
2 months ago
354
 jmp_buf controller; // to return from Coroutine_Run
11 months ago
355
 jmp_buf chunk_allocated;// for chunk allocation
2 months ago
356
 size_t size_to_retain; // for Chunk_Split
11 months ago
4
357
358
 // singletons
359
 Coroutine *tip; // top of stack chunk
360
 Coroutine *active; // currently running coroutine
2 months ago
361
 Coroutine *primary; // Coroutine_Run coroutine
9 months ago
362
 unsigned char *stack_limit; // when not NULL, where the stack finishes
11 months ago
4
363
2 months ago
364
 Coroutine *spare; // spare struct Coroutine (instead of having to malloc & fail)
365
11 months ago
4
366
 // lists
8 months ago
367
 List_Head all; // all Coroutines (in address order)
368
 List_Head free; // free Coroutines
11 months ago
4
369
 List_Head inactive; // idle or complete
370
 List_Head runable; // running or waiting to run
371
 List_Head waiting; // yielded / waiting to run
10 months ago
372
 _Cor_Mutex waiting_mutex;
11 months ago
4
373
2 months ago
374
 Coroutine *root; // The coroutine for the thread which started Coroutines
375
11 months ago
376
 // Summary of the system
377
 Coroutine_Report report;
378
11 months ago
4
379
 // state
380
 char state;
2 months ago
381
382
 int sequence;
11 months ago
4
383
 };
384
8 months ago
385
 _Cor_thread_local Coroutines *g_c;
7 months ago
386
 _Cor_thread_local unsigned char *g_stack_limit;
11 months ago
4
387
4 days ago
388
 static void ReserveStackSpace(Coroutines *cors, Coroutine *parent, size_t chunk_size, unsigned char *childs_limit, unsigned char *childs_guard);
389
 static void stack_chunk_base(Coroutines *cors, Coroutine *parent, unsigned char *prev_limit, unsigned char *limit, unsigned char *guard);
11 months ago
4
390
11 months ago
391
2 months ago
392
 static size_t
393
 Coroutine_Size
394
 (
395
 Coroutine *cor
396
 ){
397
 if (cor->limit){
398
 return (size_t)StackPointerDiff(cor->limit, cor->base);
399
 } else {
400
 return SIZE_MAX;
401
 }
402
 }
403
404
405
 static size_t
2 months ago
406
 TrimmableSize(
407
 Coroutine *cor
408
 ){
2 months ago
409
 size_t current_used = StackPointerDiff(cor->stack_top, cor->base);
410
 size_t min_size = current_used + cor->min_headroom;
411
 if (min_size < cor->min_size){
412
 min_size = cor->min_size;
2 months ago
413
 }
414
415
 if (cor->limit){
4 days ago
416
 // If this coroutine's stack chunk does not have enough space for:
417
 // * itself (min_size)
418
 // * a potential new coroutine (COROUTINE_MINIMUM_STACK_SIZE)
419
 // then return 0, ie not trimable
2 months ago
420
 if (Coroutine_Size(cor) < min_size + COROUTINE_MINIMUM_STACK_SIZE){
2 months ago
421
 return 0;
422
 }
423
 }
424
 return min_size;
425
 }
426
427
9 months ago
428
 #define GUARD_PATTERN_SIZE (4)
2 months ago
429
 /// @brief Checks whether a guard pattern is OK
430
 /// @param guard The address of the stack base end of the pattern
431
 /// @return true pattern is ok; false pattern is broken
3 months ago
432
 static inline bool
433
 Guard_Pattern_OK(
11 months ago
434
 unsigned char *guard
435
 ){
9 months ago
436
 return !guard ||
2 months ago
437
 (*StackPointerAdd(guard, 0) == 0xde &&
438
 *StackPointerAdd(guard, 1) == 0xad &&
439
 *StackPointerAdd(guard, 2) == 0xbe &&
440
 *StackPointerAdd(guard, 3) == 0xef);
11 months ago
441
 }
442
443
2 months ago
444
 /// @brief Writes a guard pattern
445
 /// @param guard Where to write the guard pattern (stack base end)
3 months ago
446
 static inline void
447
 Apply_Guard(unsigned char *guard){
2 months ago
448
 *StackPointerAdd(guard, 0) = 0xde;
449
 *StackPointerAdd(guard, 1) = 0xad;
450
 *StackPointerAdd(guard, 2) = 0xbe;
451
 *StackPointerAdd(guard, 3) = 0xef;
9 months ago
452
 }
453
454
6 months ago
455
 #ifndef NDEBUG
2 months ago
456
 /// @brief Checks a list of Coroutines to see whether it's OK
457
 /// @param head The list to check
458
 /// @param state1 One of the states Coroutines in the list are allowed
459
 /// @param state2 One of the states Coroutines in the list are allowed
460
 /// @return Coroutine_OK the is OK; other the list is broken
3 months ago
461
 static Coroutine_Err
462
 CheckListIntegrity(
463
 List_Head *head,
464
 Coroutine_State state1,
465
 Coroutine_State state2
466
 ){
6 months ago
467
 for (List_Link *link = List_Begin(head); Link_NextIsLink(link); link = Link_Next(link)){
468
 Coroutine *candidate = List_Link_Container(Coroutine, link, link);
6 months ago
469
 if (candidate->coroutines != g_c){
470
 return Coroutine_Err_InternalInsistency;
471
 }
472
 if(candidate->state != state1 && candidate->state != state2){
473
 return Coroutine_Err_InternalInsistency;
474
 }
6 months ago
475
 bool found = false;
476
 for (List_Link *link = List_Begin(&g_c->all); Link_NextIsLink(link); link = Link_Next(link)){
477
 Coroutine *candidate2 = List_Link_Container(Coroutine, all_link, link);
478
 if (candidate == candidate2){
479
 found = true;
480
 }
481
 }
6 months ago
482
 if (!found){
483
 return Coroutine_Err_InternalInsistency;
484
 }
6 months ago
485
 }
6 months ago
486
 return Coroutine_OK;
6 months ago
487
 }
488
489
2 months ago
490
 /// @brief Check the integrity of this thread's coroutine system
491
 /// @return Coroutine_OK everything's OK; other something was wrong
3 months ago
492
 static Coroutine_Err
2 months ago
493
 Coroutine_CheckIntegrity_(
3 months ago
494
 void
495
 ){
6 months ago
496
 Coroutine_Err err;
3 months ago
497
 err = CheckListIntegrity(&g_c->free, Coroutine_Free, Coroutine_Free);
6 months ago
498
 if (err){
499
 return err;
500
 }
3 months ago
501
 err = CheckListIntegrity(&g_c->inactive, Coroutine_Idle, Coroutine_Complete);
6 months ago
502
 if (err){
503
 return err;
504
 }
3 months ago
505
 err = CheckListIntegrity(&g_c->runable, Coroutine_Running, Coroutine_Running);
6 months ago
506
 if (err){
507
 return err;
508
 }
3 months ago
509
 err = CheckListIntegrity(&g_c->waiting, Coroutine_Waiting, Coroutine_Waiting);
6 months ago
510
 return err;
6 months ago
511
 }
6 months ago
512
 #endif
513
514
2 months ago
515
 /// @brief Check whether the stack has overrun
516
 /// @return Coroutine_OK the stack hasn't; other it has
3 months ago
517
 static Coroutine_Err
2 months ago
518
 Coroutine_StackHasOverrun(
3 months ago
519
 void
520
 ){
3 months ago
521
 unsigned char *stack_top = (unsigned char *)StackTopNow();
8 months ago
522
 unsigned char *stack_limit = g_c ? g_c->stack_limit : NULL;
2 months ago
523
 if (stack_limit && StackPointerDiff(stack_limit, stack_top) < 0){
9 months ago
524
 // current stack top is beyond limit - we are overrunning NOW
6 months ago
525
 return Coroutine_Err_StackOverrun;
9 months ago
526
 }
8 months ago
527
 Coroutine *me = g_c ? g_c->active : NULL;
9 months ago
528
 if (!me){
6 months ago
529
 return Coroutine_OK;
9 months ago
530
 }
6 months ago
531
 #if COROUTINE_CHECK_INTEGRITY_ON_STACK_CHECK
532
 // Check all coroutines integrity
2 months ago
533
 Coroutine_Err err = Coroutine_CheckIntegrity_();
6 months ago
534
 if (err){
535
 return err;
536
 }
6 months ago
537
 #endif
9 months ago
538
 if (me->guard){
2 months ago
539
 if (StackPointerDiff(me->guard, stack_top) < 0){
2 months ago
540
 // Stack top beyond active stack limit
2 months ago
541
 return Coroutine_Err_StackOverrun;
542
 }
543
 if (!Guard_Pattern_OK(me->guard)){
2 months ago
544
 // Guard pattern trampled
2 months ago
545
 return Coroutine_Err_StackOverrun;
3 months ago
546
 }
9 months ago
547
 }
2 months ago
548
 return Coroutine_OK;
9 months ago
549
 }
550
3 months ago
551
 #ifndef NDEBUG
2 months ago
552
 /// @brief Check system integrity - does stack overrun check and internals check
553
 /// @return Coroutine_OK all is OK; other something was wrong
3 months ago
554
 Coroutine_Err
2 months ago
555
 Coroutine_CheckIntegrity(
3 months ago
556
 void
557
 ){
2 months ago
558
 Coroutine_Err err = Coroutine_StackHasOverrun();
3 months ago
559
 #if !COROUTINE_CHECK_INTEGRITY_ON_STACK_CHECK
560
 if (!err && g_c){
2 months ago
561
 err = Coroutine_CheckIntegrity_();
3 months ago
562
 }
563
 #endif
564
 return err;
565
 }
566
 #endif
9 months ago
567
3 months ago
568
3 months ago
569
 static void
570
 ReserveStackSpace(
8 months ago
571
 Coroutines *cors,
9 months ago
572
 Coroutine *parent,
8 months ago
573
 size_t chunk_size,
4 days ago
574
 unsigned char *childs_limit,
575
 unsigned char *childs_guard
11 months ago
576
 ){
9 months ago
577
 unsigned char *chunk_of_stack = alloca(chunk_size);
2 months ago
578
 unsigned char *limit = StackLimitEnd(chunk_of_stack, chunk_of_stack+chunk_size);
579
 unsigned char *guard = StackPointerAdd(limit, -GUARD_PATTERN_SIZE);
9 months ago
580
 #if COROUTINE_RECORD_LOWEST_HEADROOM
581
 for (size_t i = 0; i <= chunk_size-GUARD_PATTERN_SIZE; i += GUARD_PATTERN_SIZE){
5 days ago
582
 Apply_Guard(StackPointerAdd(guard, -(ptrdiff_t)i));
9 months ago
583
 }
584
 #else
2 months ago
585
 Apply_Guard(guard);
9 months ago
586
 #endif
8 months ago
587
 if (parent){
2 months ago
588
 parent->limit = limit;
589
 parent->guard = guard;
8 months ago
590
 }
4 days ago
591
 stack_chunk_base(cors, parent, limit, childs_limit, childs_guard);
11 months ago
4
592
 }
593
11 months ago
594
3 months ago
595
 static void
596
 stack_chunk_base(
8 months ago
597
 Coroutines *cors,
8 months ago
598
 Coroutine *parent,
8 months ago
599
 unsigned char *prev_limit,
4 days ago
600
 unsigned char *limit,
601
 unsigned char *guard
11 months ago
602
 ){
2 months ago
603
 MyAssert(cors->spare);
604
 Coroutine *here = cors->spare;
605
 cors->spare = NULL;
606
 here->coroutines = cors;
607
 here->sequence = cors->sequence++;
608
 here->state = Coroutine_Free;
609
 here->base = prev_limit;
610
 here->limit = limit;
4 days ago
611
 here->guard = guard;
2 months ago
612
 here->stack_top = (unsigned char *)StackTopNow();
8 months ago
613
8 months ago
614
 // insert into all list
615
 if (parent){
2 months ago
616
 Link_AddAfter(&here->all_link, &parent->all_link);
8 months ago
617
 } else {
2 months ago
618
 List_AddHead(&cors->all, &here->all_link);
8 months ago
619
 }
8 months ago
620
 // add to free list
2 months ago
621
 List_AddTail(&cors->free, &here->link);
8 months ago
622
623
 cors->report.coroutines_pool_size += 1;
624
2 months ago
625
 if (!cors->tip || !Link_NextIsLink(Link_Next(&here->all_link))){
626
 cors->tip = here;
8 months ago
627
 }
628
9 months ago
629
 for(;;){
2 months ago
630
 switch (setjmp(here->buf)) {
9 months ago
631
 case Chunk_Initial:
2 months ago
632
 ready_jmp_buf(here->buf);
633
 if (here->state == Coroutine_Free){
9 months ago
634
 // return to the coroutine allocator
8 months ago
635
 longjmp(cors->chunk_allocated, 1);
9 months ago
636
 } else {
2 months ago
637
 MyAssert(here->state == Coroutine_Complete);
9 months ago
638
 // we finish here to ensure the setjmp is redone
2 months ago
639
 if (cors->primary == here) {
2 months ago
640
 // if primary coroutine - return to Coroutine_Run
8 months ago
641
 longjmp(cors->controller, Coroutines_CoroutineComplete);
9 months ago
642
 }
8 months ago
643
 _Cor_Mutex_Unlock(&cors->mutex);
2 months ago
644
 Coroutine_RunNext();
9 months ago
645
 }
5 months ago
646
 MyAssert(false);
647
 break;
8 months ago
648
 case Chunk_Split:
2 months ago
649
 // Request to split this idle block into two
650
 // g_c->size_to_retain will be set to our shorter size
4 days ago
651
 ReserveStackSpace(here->coroutines, here, g_c->size_to_retain, here->limit, here->guard);
6 months ago
652
 MyAssert(false);
5 months ago
653
 break;
9 months ago
654
 case Chunk_Enter:
655
 // request to start a coroutine (ie use the chunk for a coroutine)
656
 // arrive here with mutex locked
2 months ago
657
 MyAssert(here->state == Coroutine_Running);
658
 here->coroutines->active = here;
2 months ago
659
 EnlargeActiveIfPossible();
8 months ago
660
 _Cor_Mutex_Unlock(&cors->mutex);
2 months ago
661
 here->value = here->start(here->entry_param);
11 months ago
662
9 months ago
663
 // check the guard
2 months ago
664
 MyAssert(Guard_Pattern_OK(here->guard));
11 months ago
665
2 months ago
666
 here->stack_top = (unsigned char *)StackTopNow();
667
 _Cor_Mutex_Lock(&here->coroutines->mutex);
2 months ago
668
 TrimActiveIfPossible();
2 months ago
669
 here->coroutines->active = NULL;
670
 MyAssert(here->state == Coroutine_Running);
671
 Link_Remove(&here->link);
672
 here->state = Coroutine_Complete;
673
 List_AddTail(&here->coroutines->inactive, &here->link);
9 months ago
674
 // Coroutine has completed
675
 // Loop round to redo the setjmp() - if this coroutine yielded, then the setjmp will
676
 // need reseting
5 months ago
677
 break;
11 months ago
4
678
 }
679
 }
680
 }
681
11 months ago
682
3 months ago
683
 static void
2 months ago
684
 Coroutine_RunNext(
3 months ago
685
 void
686
 ){
11 months ago
687
 // arrive here with mutex unlocked
8 months ago
688
 _Cor_Mutex_Lock(&g_c->waiting_mutex);
689
 _Cor_Mutex_Lock(&g_c->mutex);
690
 Coroutine *next = List_Link_Container(Coroutine, link, List_GetHead(&g_c->runable));
3 months ago
691
 MyAssert(next->state == Coroutine_Running);
11 months ago
692
 longjmp(next->buf, Chunk_Enter);
6 months ago
693
 MyAssert(false);
11 months ago
694
 }
695
696
2 months ago
697
 /// @brief Ensures there's a spare Coroutine for cors
698
 /// @param cors The Coroutines to ensure a spare for
699
 /// @return true there is a spare; false there isn't
700
 static inline bool
701
 EnsureSpare
702
 (
703
 Coroutines *cors
704
 ){
705
 if (cors->spare){
706
 return true;
707
 }
708
 cors->spare = malloc(sizeof(*cors->spare));
709
 return cors->spare != NULL;
710
 }
711
712
3 months ago
713
 static Coroutine_Err
714
 Coroutines_ctor(
2 months ago
715
 Coroutines *cors,
716
 size_t min_size,
717
 size_t min_headroom
3 months ago
718
 ){
6 months ago
719
 if (_Cor_Mutex_ctor(&cors->mutex)){
2 months ago
720
 goto error;
6 months ago
721
 }
8 months ago
722
 cors->primary = NULL;
7 months ago
723
 cors->stack_limit = g_stack_limit;
11 months ago
4
724
8 months ago
725
 List_Init(&cors->all);
8 months ago
726
 List_Init(&cors->free);
727
 List_Init(&cors->inactive);
728
 List_Init(&cors->runable);
729
 List_Init(&cors->waiting);
6 months ago
730
 if (_Cor_Mutex_ctor(&cors->waiting_mutex)){
2 months ago
731
 goto error1;
6 months ago
732
 }
733
 if (_Cor_Mutex_Lock(&cors->waiting_mutex)){
2 months ago
734
 goto error2;
6 months ago
735
 }
11 months ago
4
736
8 months ago
737
 cors->report.coroutines_created = 0;
738
 cors->report.coroutines_pool_size = 0;
739
 cors->report.largest_stack = 0;
2 months ago
740
 cors->spare = NULL;
741
 cors->sequence = 0;
8 months ago
742
2 months ago
743
 Coroutine *cor = malloc(sizeof(*cor));
744
 cor->min_size = min_size;
745
 cor->min_headroom = min_headroom;
746
 cor->coroutines = cors;
747
 cor->sequence = cors->sequence++;
748
 cor->state = Coroutine_Running;
749
 cor->base = (unsigned char *)&cor;
Last month
750
 cor->chain_root = cor;
751
 cor->chain_tip = cor;
2 months ago
752
 if (cors->stack_limit){
753
 cor->limit = cors->stack_limit;
4 days ago
754
 #if COROUTINE_GUARD_AT_C_STACK_LIMIT
2 months ago
755
 cor->guard = StackPointerAdd(cor->limit, -GUARD_PATTERN_SIZE);
756
 Apply_Guard(cor->guard);
4 days ago
757
 #else
758
 cor->guard = NULL;
759
 #endif
2 months ago
760
 } else {
761
 cor->limit = cor->guard = NULL;
11 months ago
4
762
 }
2 months ago
763
 cors->root = cor;
764
 List_AddHead(&cors->all, &cor->all_link);
765
 List_AddTail(&cors->runable, &cor->link);
766
 cors->tip = cors->active = cor;
11 months ago
767
2 months ago
768
 cors->report.coroutines_pool_size += 1;
8 months ago
769
770
 cors->state = Coroutines_Started;
6 months ago
771
 return Coroutine_OK;
2 months ago
772
 error2:
773
 _Cor_Mutex_dtor(&cors->waiting_mutex);
774
 error1:
775
 _Cor_Mutex_dtor(&cors->mutex);
776
 error:
777
 return Coroutine_Err_CouldNotInitialiseSystem;
6 months ago
778
 }
8 months ago
779
3 months ago
780
 static void
781
 Coroutines_dtor(
782
 Coroutines *cors
783
 ){
6 months ago
784
 _Cor_Mutex_Lock(&cors->mutex);
785
 cors->state = Coroutines_Stopping;
786
6 months ago
787
 MyAssert(List_IsEmpty(&cors->inactive));
2 months ago
788
789
 // free the spare
790
 if (cors->spare){
791
 free(cors->spare);
792
 }
793
794
 // free all non-spares
795
 List_Link *link;
796
 List_Link *next;
797
 for (link = List_Begin(&cors->all); Link_NextIsLink(link); link = next){
798
 next = Link_Next(link);
799
 Coroutine *cor = List_Link_Container(Coroutine, all_link, link) ;
800
 free(cor);
801
 }
802
6 months ago
803
 _Cor_Mutex_Unlock(&cors->waiting_mutex);
804
 _Cor_Mutex_dtor(&cors->waiting_mutex);
805
6 months ago
806
 MyAssert(cors->state == Coroutines_Stopping);
6 months ago
807
 _Cor_Mutex_Unlock(&cors->mutex);
808
 _Cor_Mutex_dtor(&cors->mutex);
11 months ago
4
809
 }
810
11 months ago
811
3 months ago
812
 Coroutine_Err
2 months ago
813
 Coroutine_RunSystem(
2 months ago
814
 size_t min_size,
815
 size_t min_headroom,
3 months ago
816
 Coroutine_SystemStart start,
817
 void *value
818
 ){
6 months ago
819
 CHECK_SYSTEM_NOT_RUNNING
820
821
 Coroutines cors;
2 months ago
822
 Coroutine_Err err = Coroutines_ctor(&cors, min_size, min_headroom);
6 months ago
823
 if (err){
824
 return err;
825
 }
826
 g_c = &cors;
2 months ago
827
 err = start(value, cors.root);
6 months ago
828
 g_c = NULL;
829
 Coroutines_dtor(&cors);
830
 return err;
831
 }
832
833
3 months ago
834
 void
2 months ago
835
 Coroutine_SetStackLimit(
3 months ago
836
 void *limit
837
 ){
2 months ago
838
 MyAssert(!limit || !g_c || !g_stack_limit || StackPointerDiff((unsigned char *)limit, (unsigned char *)g_stack_limit) <= 0);
7 months ago
839
 g_stack_limit = limit;
840
 if (g_c){
841
 g_c->stack_limit = limit;
2 months ago
842
 if (g_c->tip){
843
 g_c->tip->limit = limit;
4 days ago
844
 #if COROUTINE_GUARD_AT_C_STACK_LIMIT
2 months ago
845
 g_c->tip->guard = StackPointerAdd(limit, -GUARD_PATTERN_SIZE);
846
 Apply_Guard(g_c->tip->guard);
4 days ago
847
 #else
848
 g_c->tip->guard = NULL;
849
 #endif
2 months ago
850
 }
7 months ago
851
 }
9 months ago
852
 }
853
854
6 months ago
855
 #if COROUTINE_RECORD_LOWEST_HEADROOM
3 months ago
856
 static size_t
2 months ago
857
 Coroutine_UpdateMinimumHeadroom(
3 months ago
858
 List_Head *list,
859
 size_t headroom
860
 ){
6 months ago
861
 for (List_Link *link = List_Begin(list); Link_NextIsLink(link); link = Link_Next(link)){
11 months ago
862
 Coroutine *cor = List_Link_Container(Coroutine, link, link);
9 months ago
863
 if (cor->guard){
2 months ago
864
 size_t chunk_size = Coroutine_Size(cor);
865
 for (size_t i = 0; i <= chunk_size-GUARD_PATTERN_SIZE; i += GUARD_PATTERN_SIZE){
5 days ago
866
 if (!Guard_Pattern_OK(StackPointerAdd(cor->guard, -(ptrdiff_t)i))){
6 months ago
867
 headroom = i < headroom ? i : headroom;
9 months ago
868
 break;
869
 }
11 months ago
870
 }
871
 }
872
 }
6 months ago
873
 return headroom;
874
 }
875
 #endif
876
877
3 months ago
878
 Coroutine_Report
2 months ago
879
 Coroutine_GetReport(
3 months ago
880
 void
881
 ){
6 months ago
882
 if (g_c){
883
 size_t headroom;
6 months ago
884
 #if COROUTINE_RECORD_LOWEST_HEADROOM
6 months ago
885
 _Cor_Mutex_Lock(&g_c->mutex);
886
 headroom = g_c->report.lowest_headroom;
2 months ago
887
 headroom = Coroutine_UpdateMinimumHeadroom(&g_c->inactive, headroom);
888
 headroom = Coroutine_UpdateMinimumHeadroom(&g_c->runable, headroom);
889
 headroom = Coroutine_UpdateMinimumHeadroom(&g_c->waiting, headroom);
6 months ago
890
 _Cor_Mutex_Unlock(&g_c->mutex);
9 months ago
891
 #else
6 months ago
892
 headroom = 0;
9 months ago
893
 #endif
6 months ago
894
 g_c->report.lowest_headroom = headroom;
11 months ago
895
6 months ago
896
 return g_c->report;
897
 } else {
898
 Coroutine_Report ret = {0, 0, 0, 0};
899
 return ret;
900
 }
6 months ago
901
 }
902
903
3 months ago
904
 struct Coroutine_Run_Params {
6 months ago
905
 Coroutine_Start start;
906
 void *value;
907
 void **result;
908
 };
909
3 months ago
910
 static Coroutine_Err
2 months ago
911
 Coroutine_Run_Starter(
2 months ago
912
 void *_params,
913
 Coroutine *root
3 months ago
914
 ){
2 months ago
915
 (void)root;
3 months ago
916
 struct Coroutine_Run_Params *params = (struct Coroutine_Run_Params *)_params;
6 months ago
917
2 months ago
918
 void *res = params->start(params->value);
919
 if (params->result){
920
 *params->result = res;
6 months ago
921
 }
2 months ago
922
923
 return Coroutine_OK;
6 months ago
924
 }
925
926
2 months ago
927
 Coroutine_Err Coroutine_Run(
2 months ago
928
 size_t min_size,
929
 size_t min_headroom,
11 months ago
930
 Coroutine_Start start,
8 months ago
931
 void *value,
932
 void **result
11 months ago
933
 ){
6 months ago
934
 if (!g_c){
2 months ago
935
 struct Coroutine_Run_Params params = {start, value, result};
2 months ago
936
 return Coroutine_RunSystem(min_size, min_headroom, Coroutine_Run_Starter, &params);
6 months ago
937
 }
938
939
 // We are in an active coroutine, so call start() directly
940
 CHECK_STACK_OVERRUN
941
 void *res = start(value);
8 months ago
942
 if (result){
6 months ago
943
 *result = res;
8 months ago
944
 }
6 months ago
945
8 months ago
946
 // no failures, so...
6 months ago
947
 return Coroutine_OK;
11 months ago
4
948
 }
949
950
2 months ago
951
 static Coroutine *Coroutine_New_Lock_Assumed(
2 months ago
952
 size_t min_size,
953
 size_t min_headroom,
8 months ago
954
 Coroutine_Start start
955
 ){
956
 List_Link *link;
957
2 months ago
958
 if (!EnsureSpare(g_c)){
959
 return NULL;
8 months ago
960
 }
961
8 months ago
962
 Coroutine *cor = NULL;
8 months ago
963
 for (link = List_Begin(&g_c->free); Link_NextIsLink(link); link = Link_Next(link)){
8 months ago
964
 Coroutine *candidate = List_Link_Container(Coroutine, link, link);
6 months ago
965
 MyAssert(candidate->coroutines == g_c);
2 months ago
966
 MyAssert(candidate->state == Coroutine_Free);
967
 if (candidate->limit){
968
 size_t candidate_size = Coroutine_Size(candidate);
969
 if (candidate_size < min_size){
970
 continue;
9 months ago
971
 }
2 months ago
972
 }
973
 // candidate has no limit, or is big enough
8 months ago
974
2 months ago
975
 // check if it's 'better' (lower in the C stack) than cor
976
 if (!cor || StackPointerDiff(candidate->base, cor->base) < 0){
8 months ago
977
 // chunk big enough, and a better choice than cor
978
 cor = candidate;
979
 }
980
 }
981
2 months ago
982
 if (!cor){
983
 return NULL;
984
 }
985
2 months ago
986
 // use this free block
2 months ago
987
 cor->min_size = min_size;
988
 cor->min_headroom = min_headroom;
989
 cor->state = Coroutine_Idle;
990
 cor->start = start;
991
 cor->value = NULL;
Last month
992
 cor->chain_root = cor;
993
 cor->chain_tip = cor;
2 months ago
994
 Link_Remove(&cor->link);
995
 List_AddHead(&g_c->inactive, &cor->link);
996
997
 // trim down to an appropriate size
998
 size_t trimmable_size = TrimmableSize(cor);
999
 if (trimmable_size){
1000
 // split block into two
1001
 if (EnsureSpare(g_c)){
1002
 g_c->size_to_retain = trimmable_size;
8 months ago
1003
 if (!setjmp(g_c->chunk_allocated)){
3 months ago
1004
 ready_jmp_buf(g_c->chunk_allocated);
8 months ago
1005
 longjmp(cor->buf, Chunk_Split);
8 months ago
1006
 }
1007
 }
11 months ago
4
1008
 }
1009
2 months ago
1010
 g_c->report.coroutines_created += 1;
8 months ago
1011
2 months ago
1012
 return cor;
1013
 }
1014
1015
1016
 static void
1017
 TrimActiveIfPossible
1018
 (
1019
 void
1020
 ){
1021
 MyAssert(g_c);
1022
1023
 // If we're the active coroutine, free the tail
1024
 Coroutine *active = g_c->active;
1025
 MyAssert(active);
1026
1027
 active->stack_top = (unsigned char *)StackTopNow();
1028
 ptrdiff_t timmablesize = TrimmableSize(active);
1029
 if (!timmablesize){
1030
 return;
8 months ago
1031
 }
2 months ago
1032
1033
 if (!EnsureSpare(g_c)){
1034
 return;
8 months ago
1035
 }
1036
2 months ago
1037
 // enough space for a second coroutine so free the unused stack space from active
1038
 if (!setjmp(g_c->chunk_allocated)){
1039
 ready_jmp_buf(g_c->chunk_allocated);
4 days ago
1040
 ReserveStackSpace(g_c, active, timmablesize - StackPointerDiff(active->stack_top, active->base), active->limit, active->guard);
2 months ago
1041
 MyAssert(false);
1042
 }
1043
 }
1044
1045
1046
 static void
1047
 EnlargeActiveIfPossible(
1048
 void
1049
 ){
1050
 MyAssert(g_c);
1051
 Coroutine *active = g_c->active;
1052
 MyAssert(active);
1053
1054
 List_Link *next = Link_Next(&active->all_link);
1055
 if (!Link_NextIsLink(next)){
1056
 return;
1057
 }
1058
 Coroutine *cor = List_Link_Container(Coroutine, all_link, next);
1059
 if (cor->state != Coroutine_Free){
1060
 return;
1061
 }
1062
1063
 // Following block is free, merge with this
1064
 active->limit = cor->limit;
1065
 active->guard = cor->guard;
8 months ago
1066
 Link_Remove(&cor->link);
2 months ago
1067
 Link_Remove(&cor->all_link);
1068
 if (g_c->tip == cor){
1069
 g_c->tip = active;
1070
 }
1071
 if (g_c->spare){
1072
 free(cor);
1073
 } else {
1074
 g_c->spare = cor;
1075
 }
8 months ago
1076
 }
11 months ago
1077
8 months ago
1078
3 months ago
1079
 Coroutine *
2 months ago
1080
 Coroutine_New(
2 months ago
1081
 size_t min_size,
1082
 size_t min_headroom,
8 months ago
1083
 Coroutine_Start start
1084
 ){
2 months ago
1085
 MyAssert(g_c && g_c->state == Coroutines_Started);
2 months ago
1086
 MyAssert(!Coroutine_StackHasOverrun());
8 months ago
1087
2 months ago
1088
 // Make the paramaters make sense
1089
 if (min_size < min_headroom){
1090
 min_size = min_headroom;
1091
 }
1092
8 months ago
1093
 _Cor_Mutex_Lock(&g_c->mutex);
1094
2 months ago
1095
 TrimActiveIfPossible();
8 months ago
1096
2 months ago
1097
 Coroutine *cor = Coroutine_New_Lock_Assumed(min_size, min_headroom, start);
2 months ago
1098
1099
 if (cor && Coroutine_Size(cor) > g_c->report.largest_stack){
1100
 g_c->report.largest_stack = Coroutine_Size(cor);
8 months ago
1101
 }
1102
2 months ago
1103
 EnlargeActiveIfPossible();
1104
8 months ago
1105
 _Cor_Mutex_Unlock(&g_c->mutex);
1106
11 months ago
4
1107
 return cor;
1108
 }
1109
11 months ago
1110
3 months ago
1111
 void
2 months ago
1112
 Coroutine_Delete(
11 months ago
1113
 Coroutine *cor
1114
 ){
2 months ago
1115
 MyAssert(!Coroutine_StackHasOverrun());
8 months ago
1116
 if (cor){
1117
 Coroutines *cors = cor->coroutines;
1118
 _Cor_Mutex_Lock(&cors->mutex);
3 months ago
1119
 MyAssert(cor->state == Coroutine_Idle || cor->state == Coroutine_Complete);
6 months ago
1120
1121
 #if COROUTINE_RECORD_LOWEST_HEADROOM
1122
 if (cor->guard){
2 months ago
1123
 unsigned char *guard = cor->guard;
1124
 ptrdiff_t chunk_size = Coroutine_Size(cor);
1125
 ptrdiff_t myheadroom;
1126
 for (myheadroom = 0; myheadroom <= chunk_size-(ptrdiff_t)GUARD_PATTERN_SIZE; myheadroom += GUARD_PATTERN_SIZE){
1127
 if (!Guard_Pattern_OK(StackPointerAdd(guard, -myheadroom))){
6 months ago
1128
 break;
1129
 }
1130
 }
2 months ago
1131
 if (myheadroom < (ptrdiff_t)g_c->report.lowest_headroom || g_c->report.lowest_headroom == 0){
6 months ago
1132
 g_c->report.lowest_headroom = myheadroom;
1133
 }
1134
 }
1135
 #endif
1136
3 months ago
1137
 cor->state = Coroutine_Free;
8 months ago
1138
 Link_Remove(&cor->link);
1139
8 months ago
1140
 // insert into free list
1141
 List_AddHead(&cors->free, &cor->link);
8 months ago
1142
1143
 // Check for merge with following Coroutine
8 months ago
1144
 List_Link *link = Link_Next(&cor->all_link);
8 months ago
1145
 if (Link_NextIsLink(link)){
8 months ago
1146
 Coroutine *listcor = List_Link_Container(Coroutine, all_link, link);
3 months ago
1147
 if (listcor->state == Coroutine_Free){
8 months ago
1148
 // merge
1149
 cor->limit = listcor->limit;
1150
 cor->guard = listcor->guard;
8 months ago
1151
 Link_Remove(&listcor->all_link);
8 months ago
1152
 Link_Remove(&listcor->link);
8 months ago
1153
 if (g_c->tip == listcor){
1154
 g_c->tip = cor;
1155
 }
2 months ago
1156
1157
 // free up the now unused struct
1158
 if (g_c->spare){
1159
 free(listcor);
1160
 } else {
1161
 g_c->spare = listcor;
1162
 }
8 months ago
1163
 }
1164
 }
1165
1166
 // check for merge with prev coroutine
8 months ago
1167
 link = Link_Prev(&cor->all_link);
8 months ago
1168
 if (Link_PrevIsLink(link)){
8 months ago
1169
 Coroutine *listcor = List_Link_Container(Coroutine, all_link, link);
3 months ago
1170
 if (listcor->state == Coroutine_Free){
8 months ago
1171
 // merge
1172
 listcor->limit = cor->limit;
1173
 listcor->guard = cor->guard;
8 months ago
1174
 Link_Remove(&cor->all_link);
8 months ago
1175
 Link_Remove(&cor->link);
8 months ago
1176
 if (g_c->tip == cor){
1177
 g_c->tip = listcor;
1178
 }
2 months ago
1179
1180
 // free up the now unused struct
1181
 if (g_c->spare){
1182
 free(cor);
1183
 } else {
1184
 g_c->spare = cor;
1185
 }
8 months ago
1186
 }
1187
 }
2 months ago
1188
1189
 EnlargeActiveIfPossible();
8 months ago
1190
8 months ago
1191
 _Cor_Mutex_Unlock(&cors->mutex);
1192
 }
11 months ago
4
1193
 }
1194
11 months ago
1195
2 months ago
1196
 // Coroutine_Continue, assuming the mutex is claimed
6 months ago
1197
 // return false for success, true for something went wrong
3 months ago
1198
 static Coroutine_Err
2 months ago
1199
 Coroutine_Continue_(
6 months ago
1200
 Coroutines *cors,
11 months ago
1201
 Coroutine *cor,
1202
 void *value,
1203
 bool early
1204
 ){
Last month
1205
 cor = cor->chain_tip;
3 months ago
1206
 if (cor->state == Coroutine_Running){
6 months ago
1207
 // already running
6 months ago
1208
 return Coroutine_OK;
6 months ago
1209
 }
3 months ago
1210
 if (cor->state != Coroutine_Idle && cor->state != Coroutine_Waiting){
6 months ago
1211
 return Coroutine_Err_WrongState;
1212
 }
11 months ago
4
1213
 cor->entry_param = value;
3 months ago
1214
 cor->state = Coroutine_Running;
8 months ago
1215
 Link_Remove(&cor->link);
11 months ago
4
1216
 if ( early ) {
11 months ago
1217
 List_AddHead(&cors->runable, &cor->link);
11 months ago
4
1218
 } else {
11 months ago
1219
 List_AddTail(&cors->runable, &cor->link);
11 months ago
4
1220
 }
10 months ago
1221
 _Cor_Mutex_Unlock(&cors->waiting_mutex);
6 months ago
1222
 return Coroutine_OK;
10 months ago
1223
 }
1224
1225
3 months ago
1226
 Coroutine_Err
2 months ago
1227
 Coroutine_Continue(
10 months ago
1228
 Coroutine *cor,
1229
 void *value,
1230
 bool early
1231
 ){
2 months ago
1232
 MyAssert(!Coroutine_StackHasOverrun());
10 months ago
1233
 Coroutines *cors = cor->coroutines;
10 months ago
1234
 _Cor_Mutex_Lock(&cors->mutex);
2 months ago
1235
 Coroutine_Err err = Coroutine_Continue_(cors, cor, value, early);
10 months ago
1236
 _Cor_Mutex_Unlock(&cors->mutex);
6 months ago
1237
 return err;
11 months ago
4
1238
 }
1239
11 months ago
1240
3 months ago
1241
 void *
2 months ago
1242
 Coroutine_Yield(
11 months ago
1243
 void *value,
3 months ago
1244
 Coroutine_YieldCallback on_yield,
11 months ago
1245
 void *yield_me
11 months ago
1246
 ){
6 months ago
1247
 MyAssert(g_c);
8 months ago
1248
 Coroutine *me = g_c->active;
6 months ago
1249
 MyAssert(me);
2 months ago
1250
 MyAssert(!Coroutine_StackHasOverrun());
11 months ago
1251
8 months ago
1252
 _Cor_Mutex_Lock(&g_c->mutex);
11 months ago
1253
 Coroutines *cors = me->coroutines;
3 months ago
1254
 MyAssert(me && me->state == Coroutine_Running && cors == g_c);
3 months ago
1255
 me->stack_top = (unsigned char *)StackTopNow();
Last month
1256
 me->value = NULL;
1257
 me->chain_root->value = value;
5 days ago
1258
 Coroutine *prev_tip = me->chain_root->chain_tip;
Last month
1259
 me->chain_root->chain_tip = me;
3 months ago
1260
 me->state = Coroutine_Waiting;
11 months ago
1261
2 months ago
1262
 TrimActiveIfPossible();
1263
8 months ago
1264
 Link_Remove(&me->link);
10 months ago
1265
 if (!List_IsEmpty(&cors->runable)){
10 months ago
1266
 _Cor_Mutex_Unlock(&cors->waiting_mutex);
10 months ago
1267
 }
11 months ago
1268
 List_AddTail(&cors->waiting, &me->link);
11 months ago
1269
11 months ago
1270
 switch (setjmp(me->buf)){
1271
 case Chunk_Initial:
3 months ago
1272
 ready_jmp_buf(me->buf);
10 months ago
1273
 _Cor_Mutex_Unlock(&cors->mutex);
11 months ago
1274
 on_yield(yield_me);
2 months ago
1275
 Coroutine_RunNext();
6 months ago
1276
 MyAssert(false);
5 months ago
1277
 break;
11 months ago
1278
 case Chunk_Enter:
1279
 // arrive here with mutex locked
11 months ago
1280
 cors->active = me;
2 months ago
1281
 MyAssert(!Coroutine_StackHasOverrun());
11 months ago
1282
 // when we return here - we are running again
3 months ago
1283
 MyAssert(me->state == Coroutine_Running);
2 months ago
1284
 EnlargeActiveIfPossible();
5 days ago
1285
 me->chain_root->chain_tip = prev_tip;
11 months ago
1286
 void *res = me->entry_param;
10 months ago
1287
 _Cor_Mutex_Unlock(&cors->mutex);
11 months ago
1288
 return res;
11 months ago
4
1289
 }
5 months ago
1290
 MyAssert(false);
11 months ago
1291
 return NULL;
11 months ago
4
1292
 }
1293
11 months ago
1294
3 months ago
1295
 void *
2 months ago
1296
 Coroutine_GetValue(
11 months ago
1297
 Coroutine *cor
1298
 ){
11 months ago
4
1299
 return cor->value;
1300
 }
1301
11 months ago
1302
3 months ago
1303
 Coroutine *
2 months ago
1304
 Coroutine_GetActive(
3 months ago
1305
 void
1306
 ){
Last month
1307
 return g_c ? g_c->active->chain_root : NULL;
11 months ago
4
1308
 }
1309
11 months ago
1310
2 months ago
1311
 ptrdiff_t
2 months ago
1312
 Coroutine_GetStackHeadroom(
3 months ago
1313
 void
1314
 ){
6 months ago
1315
 Coroutine *me = g_c ? g_c->active : NULL;
4 days ago
1316
 if (me){
1317
 // active coroutine
1318
 if (me->guard){
1319
 return StackPointerDiff(me->guard, (unsigned char *)StackTopNow());
1320
 } else if (me->limit) {
1321
 return StackPointerDiff(me->limit, (unsigned char *)StackTopNow());
1322
 }
1323
 } else {
9 months ago
1324
 // no active coroutine
6 months ago
1325
 if (g_stack_limit){
2 months ago
1326
 return StackPointerDiff(g_stack_limit, (unsigned char *)StackTopNow());
9 months ago
1327
 }
1328
 }
4 days ago
1329
 // The biggest ptrdiff_t possible
1330
 return PTRDIFF_MAX;
11 months ago
1331
 }
1332
1333
3 months ago
1334
 void *
2 months ago
1335
 Coroutine_GetStackHWM(
3 months ago
1336
 void
1337
 ){
6 months ago
1338
 MyAssert(g_c);
2 months ago
1339
 MyAssert(g_c->state == Coroutines_Started);
2 months ago
1340
 MyAssert(!Coroutine_StackHasOverrun());
9 months ago
1341
 // Find where the guards end
2 months ago
1342
 unsigned char *guard = g_c->active->guard;
1343
 if (guard){
2 months ago
1344
 ptrdiff_t headroom = Coroutine_GetStackHeadroom();
2 months ago
1345
 for (ptrdiff_t i = 0; i <= headroom; i += GUARD_PATTERN_SIZE){
1346
 if (!Guard_Pattern_OK(StackPointerAdd(guard, -i))){
1347
 return StackPointerAdd(guard, i);
1348
 }
1349
 }
9 months ago
1350
 }
1351
 return guard;
1352
 }
10 months ago
1353
1354
3 months ago
1355
 void
2 months ago
1356
 Coroutine_ClearStackForHWM(
3 months ago
1357
 void
1358
 ){
6 months ago
1359
 MyAssert(g_c);
2 months ago
1360
 MyAssert(!Coroutine_StackHasOverrun());
2 months ago
1361
 unsigned char *guard = g_c->active->guard;
1362
 if (guard){
2 months ago
1363
 ptrdiff_t headroom = Coroutine_GetStackHeadroom();
2 months ago
1364
 for (ptrdiff_t i = 0; i <= headroom; i += GUARD_PATTERN_SIZE){
1365
 Apply_Guard(StackPointerAdd(guard, -i));
1366
 }
9 months ago
1367
 }
1368
 }
1369
1370
3 months ago
1371
 static bool
2 months ago
1372
 Coroutine_CanStartCoroutine_Lock_Assumed(
2 months ago
1373
 size_t min_size
8 months ago
1374
 ){
1375
 if (!g_c->stack_limit){
9 months ago
1376
 return true;
1377
 }
9 months ago
1378
2 months ago
1379
 // check free list
8 months ago
1380
 List_Link *link;
1381
 for (link = List_Begin(&g_c->free); Link_NextIsLink(link); link = Link_Next(link)){
1382
 Coroutine *cor = List_Link_Container(Coroutine, link, link);
2 months ago
1383
 size_t cor_size = Coroutine_Size(cor);
1384
 if (cor_size >= min_size){
8 months ago
1385
 return true;
1386
 }
1387
 }
1388
2 months ago
1389
 // Check if this coroutine has enough size
1390
 Coroutine *me = g_c->active;
1391
 me->stack_top = (unsigned char *)StackTopNow();
1392
 ptrdiff_t reduced_size = TrimmableSize(g_c->active);
1393
 if (reduced_size && StackPointerDiff(g_c->active->limit, g_c->active->base) - reduced_size > (ptrdiff_t)min_size){
1394
 return true;
1395
 }
1396
8 months ago
1397
 return false;
9 months ago
1398
 }
1399
3 months ago
1400
 bool
2 months ago
1401
 Coroutine_CanStartCoroutine(
8 months ago
1402
 size_t size
1403
 ){
2 months ago
1404
 MyAssert(g_c && g_c->state == Coroutines_Started);
2 months ago
1405
 MyAssert(!Coroutine_StackHasOverrun());
6 months ago
1406
8 months ago
1407
 _Cor_Mutex_Lock(&g_c->mutex);
1408
2 months ago
1409
 bool result = Coroutine_CanStartCoroutine_Lock_Assumed(size);
8 months ago
1410
1411
 _Cor_Mutex_Unlock(&g_c->mutex);
1412
1413
 return result;
1414
 }
1415
2 months ago
1416
 static bool
1417
 Coroutine_GetUsefulFreeSpace_Lock_Assumed(
2 months ago
1418
 size_t min_size,
2 months ago
1419
 size_t overhead
1420
 ){
1421
 if (!g_c->stack_limit){
1422
 return SIZE_MAX;
1423
 }
1424
1425
 size_t total = 0;
1426
1427
 // check free list
1428
 List_Link *link;
1429
 for (link = List_Begin(&g_c->free); Link_NextIsLink(link); link = Link_Next(link)){
1430
 Coroutine *cor = List_Link_Container(Coroutine, link, link);
1431
 size_t cor_size = Coroutine_Size(cor);
2 months ago
1432
 if (cor_size >= min_size){
2 months ago
1433
 total += cor_size - overhead;
1434
 }
1435
 }
1436
1437
 return total;
1438
 }
1439
1440
 size_t
1441
 Coroutine_GetUsefulFreeSpace(
2 months ago
1442
 size_t min_size,
2 months ago
1443
 size_t overhead
1444
 ){
1445
 MyAssert(g_c && g_c->state == Coroutines_Started);
1446
 MyAssert(!Coroutine_StackHasOverrun());
1447
2 months ago
1448
 if (min_size < overhead){
1449
 min_size = overhead;
1450
 }
1451
2 months ago
1452
 _Cor_Mutex_Lock(&g_c->mutex);
1453
2 months ago
1454
 size_t result = Coroutine_GetUsefulFreeSpace_Lock_Assumed(min_size, overhead);
2 months ago
1455
1456
 _Cor_Mutex_Unlock(&g_c->mutex);
1457
1458
 return result;
1459
 }
1460
3 months ago
1461
 void *
2 months ago
1462
 Coroutine_GetCStackTop(
3 months ago
1463
 void
1464
 ){
2 months ago
1465
 if (!g_c || g_c->tip == g_c->active){
1466
 return (void *)StackTopNow();
9 months ago
1467
 }
2 months ago
1468
1469
 return g_c->tip->stack_top;
11 months ago
1470
 }
1471
1472
3 months ago
1473
 // Inspired by cpython...
1474
 #ifdef __has_builtin
1475
 # define Coroutine__has_builtin(x) __has_builtin(x)
1476
 #else
1477
 # define Coroutine__has_builtin(x) 0
1478
 #endif
1479
1480
 #if !Coroutine__has_builtin(__builtin_frame_address) && !defined(__GNUC__) && !defined(_MSC_VER)
1481
 static uintptr_t return_pointer_as_int(char* p) {
1482
 return (uintptr_t)p;
9 months ago
1483
 }
3 months ago
1484
 #endif
9 months ago
1485
3 months ago
1486
 static inline uintptr_t
1487
 StackTopNow(void) {
1488
 #if Coroutine__has_builtin(__builtin_frame_address) || defined(__GNUC__)
1489
 return (uintptr_t)__builtin_frame_address(0);
1490
 #elif defined(_MSC_VER)
1491
 return (uintptr_t)_AddressOfReturnAddress();
1492
 #else
1493
 char here;
1494
 /* Avoid compiler warning about returning stack address */
1495
 return return_pointer_as_int(&here);
1496
 #endif
1497
 }
1498
 // ...inspired by cpython
9 months ago
1499
3 months ago
1500
3 months ago
1501
 struct Coroutine_ChainParam {
11 months ago
1502
 Coroutine_Start start;
1503
 void *value;
1504
 Coroutine *ret;
1505
 };
1506
1507
3 months ago
1508
 static void *
2 months ago
1509
 Coroutine_ChainFn(
11 months ago
1510
 void *param
1511
 ){
Last month
1512
 struct Coroutine_ChainParam params = *(struct Coroutine_ChainParam *)param;
1513
 void *res = params.start(params.value);
1514
 params.ret->chain_tip = params.ret;
1515
 return (void *)(uintptr_t)Coroutine_Continue(params.ret, res, true);
11 months ago
1516
 }
1517
1518
3 months ago
1519
 static void
2 months ago
1520
 Coroutine_ChainYield(
11 months ago
1521
 void *unused
1522
 ){
1523
 (void)unused;
1524
 }
1525
1526
3 months ago
1527
 Coroutine_Err
2 months ago
1528
 Coroutine_Chain(
2 months ago
1529
 size_t min_size,
1530
 size_t min_headroom,
11 months ago
1531
 Coroutine_Start start,
8 months ago
1532
 void *value,
1533
 void **result
11 months ago
1534
 ){
2 months ago
1535
 MyAssert(!Coroutine_StackHasOverrun());
1536
 Coroutine *cor = Coroutine_New(min_size, min_headroom, Coroutine_ChainFn);
8 months ago
1537
 if (!cor){
1538
 // failed
6 months ago
1539
 return Coroutine_Err_NoStack;
8 months ago
1540
 }
Last month
1541
 cor->chain_root = g_c->active->chain_root;
1542
 g_c->active->chain_tip = cor;
3 months ago
1543
 struct Coroutine_ChainParam params = {
11 months ago
1544
 start,
1545
 value,
Last month
1546
 g_c->active,
11 months ago
1547
 };
2 months ago
1548
 Coroutine_Err err = Coroutine_Continue(cor, &params, true);
2 months ago
1549
 if (!err){
1550
 void *res = Coroutine_Yield(NULL, Coroutine_ChainYield, NULL);
1551
 err = (Coroutine_Err)(uintptr_t)Coroutine_GetValue(cor);
1552
 if (!err && result){
1553
 *result = res;
1554
 }
6 months ago
1555
 }
2 months ago
1556
 Coroutine_Delete(cor);
6 months ago
1557
 // success! ...probably
1558
 return err;
11 months ago
1559
 }
1560
1561
2 months ago
1562
 static Coroutine *
1563
 Coroutine_BiggestFreeBlock_Lock_Assumed(
Last month
1564
 void
2 months ago
1565
 ){
1566
 // check free list
1567
 Coroutine *best = NULL;
1568
 size_t best_size = 0;
1569
 List_Link *link;
1570
 for (link = List_Begin(&g_c->free); Link_NextIsLink(link); link = Link_Next(link)){
1571
 Coroutine *cor = List_Link_Container(Coroutine, link, link);
1572
 size_t cor_size = Coroutine_Size(cor);
1573
 if (cor_size > best_size){
1574
 best = cor;
1575
 best_size = cor_size;
1576
 }
1577
 }
1578
1579
 return best;
1580
 }
1581
1582
1583
 Coroutine_Err
1584
 Coroutine_CallWithMaxStack(
1585
 Coroutine_Start start,
1586
 void *value,
1587
 void **result
1588
 ){
1589
 MyAssert(!Coroutine_StackHasOverrun());
1590
 size_t headroom = (size_t)Coroutine_GetStackHeadroom();
1591
 if (headroom != PTRDIFF_MAX){
1592
 _Cor_Mutex_Lock(&g_c->mutex);
1593
1594
 Coroutine *cor = Coroutine_BiggestFreeBlock_Lock_Assumed();
1595
1596
 if (cor && Coroutine_Size(cor) > headroom){
1597
 // use cor for chaining
1598
 Coroutine *active = Coroutine_GetActive();
1599
 cor->min_size = active->min_size;
1600
 cor->min_headroom = active->min_headroom;
1601
 cor->state = Coroutine_Idle;
1602
 cor->start = Coroutine_ChainFn;
1603
 cor->value = NULL;
Last month
1604
 cor->chain_root = g_c->active->chain_root;
1605
 cor->chain_tip = cor;
1606
 g_c->active->chain_tip = cor;
2 months ago
1607
 Link_Remove(&cor->link);
1608
 List_AddHead(&g_c->inactive, &cor->link);
1609
 g_c->report.coroutines_created += 1;
1610
 } else {
1611
 cor = NULL;
1612
 }
1613
1614
 _Cor_Mutex_Unlock(&g_c->mutex);
1615
1616
 if (cor){
1617
 // we're chaining, not calling
1618
 struct Coroutine_ChainParam params = {
1619
 start,
1620
 value,
1621
 g_c->active
1622
 };
1623
 Coroutine_Err err = Coroutine_Continue(cor, &params, true);
1624
 if (!err){
1625
 void *res = Coroutine_Yield(NULL, Coroutine_ChainYield, NULL);
1626
 err = (Coroutine_Err)(uintptr_t)Coroutine_GetValue(cor);
1627
 if (!err && result){
1628
 *result = res;
1629
 }
1630
 }
1631
 Coroutine_Delete(cor);
1632
 return err;
1633
 }
1634
 }
1635
 void *ret = start(value);
1636
 if (result){
1637
 *result = ret;
1638
 }
1639
 return Coroutine_OK;
1640
 }
1641
1642
3 months ago
1643
 bool
2 months ago
1644
 Coroutine_IsRunning(
11 months ago
1645
 Coroutine *cor
3 months ago
1646
 ){
Last month
1647
 int state = cor->chain_tip->state;
3 months ago
1648
 return state == Coroutine_Running || state == Coroutine_Waiting;
11 months ago
4
1649
 }
11 months ago
1650
1651
2 months ago
1652
 bool Coroutine_IsComplete(
9 months ago
1653
 Coroutine *cor
3 months ago
1654
 ){
Last month
1655
 int state = cor->chain_tip->state;
3 months ago
1656
 return state == Coroutine_Complete;
9 months ago
1657
 }
1658
1659
3 months ago
1660
 bool
2 months ago
1661
 Coroutine_IsStarted(
3 months ago
1662
 void
1663
 ){
2 months ago
1664
 return g_c && g_c->state == Coroutines_Started;
11 months ago
1665
 }
8 months ago
1666
3 months ago
1667
 void
2 months ago
1668
 Coroutine_Dump_(
3 months ago
1669
 void
1670
 ){
8 months ago
1671
 char *state_to_text[] = {
1672
 "Free",
1673
 "Idle",
1674
 "Running",
1675
 "Waiting",
1676
 "Complete"
1677
 };
1678
 unsigned idx = 0;
1679
 List_Link *link;
1680
 for (link = List_Begin(&g_c->all); Link_NextIsLink(link); link = Link_Next(link)){
1681
 Coroutine *cor = List_Link_Container(Coroutine, all_link, link);
2 months ago
1682
 printf("%d) %p %p %zu (%s) %s\n", idx++, cor, cor->base, StackPointerDiff(cor->limit, cor->base), state_to_text[cor->state], cor == g_c->tip ? " (TIP)" : "");
8 months ago
1683
 }
1684
 }
2 months ago
1685
 #include "coroutine_names_undef.h"
1686