0 branches 0 tags
106
Fix signed/unsigned vulnerability and chain_tip management
jonroach 4 Aug 2026 13:18
1 contributor
1679 lines41.8 KB
Newer
Older
Text Raw Blame
  • Delete
-
+
commited
{line.log.rev}
 on
11 months ago
4
1
 #include "coroutine.h"
2
 #include <assert.h>
3
 #include <setjmp.h>
4
 #include <stdbool.h>
5
 #include <stddef.h>
8 months ago
6
 #include <stdio.h>
2 months ago
7
 #include <stdlib.h>
10 months ago
8
 #include "cor_platform.h"
11 months ago
4
9
2 months ago
10
 #include "coroutine_names_def.h"
11
9 months ago
12
 // see CPython again, this time from ctypes.h
13
 #if (defined (__SVR4) && defined (__sun)) || defined(COROUTINE_HAVE_ALLOCA_H)
14
 # include <alloca.h>
15
 #elif defined(MS_WIN32)
16
 # include <malloc.h>
17
 #endif
11 months ago
4
18
9 months ago
19
 /* If the system does not define alloca(), we have to hope for a compiler builtin. */
20
 #ifndef alloca
21
 # if defined __GNUC__ || (__clang_major__ >= 4)
22
 # define alloca __builtin_alloca
23
 # else
24
 # error "Could not define alloca() on your platform."
25
 # endif
26
 #endif
27
6 months ago
28
 typedef struct Coroutines Coroutines;
29
2 months ago
30
 static void Coroutine_RunNext(void);
31
 static Coroutine_Err Coroutine_Continue_(Coroutines *cors, Coroutine *cor, void *value, bool early);
3 months ago
32
 static uintptr_t StackTopNow(void);
2 months ago
33
 static void TrimActiveIfPossible(void);
34
 static void EnlargeActiveIfPossible(void);
11 months ago
35
6 months ago
36
 #ifndef NDEBUG
37
 // In debug builds, use the built-in assert
38
 #define MyAssert assert
39
 #else
40
 #if 1
41
 // In non-debug builds, normally use this - all the asserts are disabled
42
 #define MyAssert(cond)
43
 #else
44
 // In non-debug builds with stack problems, you can use this.
45
 // This activates all the asserts, and gives a line to put a
46
 // breakpoint in your debugger.
47
 static void _MyAssert(bool cond, char const *msg)
48
 {
49
 if (!cond){
50
 fputs("Assertion failed: ", stdout);
51
 fputs(msg, stdout);
52
 fputs("\n", stdout);
53
 }
54
 }
55
 #define MyAssert(cond) _MyAssert(cond, #cond)
56
 #endif
57
 #endif
58
2 months ago
59
 static inline ptrdiff_t
60
 StackPointerDiff
61
 (
62
 unsigned char *a,
63
 unsigned char *b
64
 ){
65
 #if COROUTINE_STACK_GROWS_UP
66
 return a - b;
67
 #else
68
 return b - a;
69
 #endif
70
 }
71
72
 static inline unsigned char *
73
 StackPointerAdd
74
 (
75
 unsigned char *a,
76
 ptrdiff_t b
77
 ){
78
 #if COROUTINE_STACK_GROWS_UP
79
 return a + b;
80
 #else
81
 return a - b;
82
 #endif
83
 }
84
85
 // Unused...
86
 // static inline unsigned char *
87
 // StackBaseEnd
88
 // (
89
 // unsigned char *memlo,
90
 // unsigned char *memhi
91
 // ){
92
 // #if COROUTINE_STACK_GROWS_UP
93
 // (void)memhi;
94
 // return memlo;
95
 // #else
96
 // (void)memlo;
97
 // return memhi-1;
98
 // #endif
99
 // }
100
 // ...unused
101
102
 static inline unsigned char *
103
 StackLimitEnd
104
 (
105
 unsigned char *memlo,
106
 unsigned char *memhi
107
 ){
108
 #if COROUTINE_STACK_GROWS_UP
109
 (void)memlo;
110
 return memhi;
111
 #else
112
 (void)memhi;
113
 return memlo-1;
114
 #endif
115
 }
116
6 months ago
117
 #define CHECK_SYSTEM_RUNNING \
118
 if (!g_c){ \
119
 return Coroutine_Err_SystemNotRunning; \
120
 }
121
 #define CHECK_SYSTEM_NOT_RUNNING \
122
 if (g_c){ \
123
 return Coroutine_Err_SystemRunning; \
124
 }
125
 #define CHECK_COROUTINE_THREAD \
126
 if (cor->coroutines != g_c){ \
127
 return Coroutine_Err_CoroutineFromWrongThread; \
128
 }
129
 #define CHECK_NO_COROUTINE_RUNNING \
130
 if (g_c->state != Coroutines_Started){ \
131
 return Coroutine_Err_ACoroutineIsAlreadyRunning; \
132
 }
133
 #define CHECK_STACK_OVERRUN \
134
 { \
2 months ago
135
 Coroutine_Err err = Coroutine_StackHasOverrun(); \
6 months ago
136
 if (err){ \
137
 return err; \
138
 } \
139
 } while (0);
140
3 months ago
141
142
 static inline void ready_jmp_buf(jmp_buf buf) {
143
 #if defined(_M_X64) || defined(_M_ARM64)
144
 // Win64:
145
 // Set Frame to 0 on Windows 64 bit to prevent C++ stack unwinding in longjmp().
146
 // Win32:
147
 // Doesn't do this, so only needed on the 2 64 bit Windows versions
148
 ((_JUMP_BUFFER*)buf)->Frame = 0;
3 months ago
149
 #else
150
 (void)buf;
3 months ago
151
 #endif
152
 }
153
11 months ago
4
154
 ///////////////////////////////////////////////////////////////////////////////
155
 // 2-way linked lists...
156
 //
11 months ago
157
 // Brought inline here to avoid namespace polution
11 months ago
4
158
 ///////////////////////////////////////////////////////////////////////////////
159
160
 typedef struct List_Link List_Link;
161
 struct List_Link {
162
 List_Link *next;
163
 List_Link *prev;
164
 };
165
166
 typedef struct List_Head List_Head;
167
 struct List_Head {
168
 union {
169
 struct {
170
 List_Link link;
171
 List_Link *filler;
172
 } fwd;
173
 struct {
174
 List_Link *filler;
175
 List_Link link;
176
 } back;
177
 };
178
 };
179
11 months ago
180
181
 static inline bool List_IsEmpty(
182
 const List_Head *list
183
 ){
11 months ago
4
184
 return list->fwd.link.next == &list->back.link;
185
 }
186
11 months ago
187
188
 static inline List_Link *List_GetHead(
189
 const List_Head *list
190
 ){
11 months ago
4
191
 return List_IsEmpty(list) ? NULL : list->fwd.link.next;
192
 }
11 months ago
193
194
8 months ago
195
 static inline List_Link *List_Begin(
196
 const List_Head *list
197
 ){
198
 return list->fwd.link.next;
199
 }
200
201
202
 static inline bool Link_NextIsLink(
203
 const List_Link *link
204
 ){
205
 return link->next != NULL;
206
 }
207
208
209
 static inline List_Link *Link_Next(
210
 List_Link *link
211
 ){
212
 return link->next;
213
 }
214
215
216
 static inline bool Link_PrevIsLink(
217
 const List_Link *link
218
 ){
219
 return link->prev != NULL;
220
 }
221
222
223
 static inline List_Link *Link_Prev(
224
 List_Link *link
225
 ){
226
 return link->prev;
227
 }
228
2 months ago
229
 // Unused...
230
 // static inline List_Link *List_GetTail(
231
 // const List_Head *list
232
 // ){
233
 // return List_IsEmpty(list) ? NULL : list->back.link.prev;
234
 // }
235
 // ...unused
11 months ago
236
237
11 months ago
4
238
 #define OFFSETOF(Container, Field) ((char *)&((Container *)4)->Field - (char *)(Container *)4)
239
 #define List_Link_Container(Container, Link, link) ((Container *)((char *)(link) - OFFSETOF(Container, Link)))
240
11 months ago
241
3 months ago
242
 static inline void
243
 List_Init(
11 months ago
244
 List_Head *list
245
 ){
11 months ago
4
246
 list->fwd.link.next = &list->back.link;
247
 list->fwd.link.prev = NULL;
248
 list->back.link.prev = &list->fwd.link;
249
 }
250
11 months ago
251
3 months ago
252
 static inline void
253
 Link_AddAfter(
8 months ago
254
 List_Link *link,
255
 List_Link *after
256
 ){
257
 link->next = after->next;
258
 link->prev = after;
259
 after->next->prev = link;
260
 after->next = link;
261
 }
262
263
3 months ago
264
 static inline void
265
 List_AddHead(
11 months ago
266
 List_Head *list,
267
 List_Link *link
268
 ){
8 months ago
269
 Link_AddAfter(link, &list->fwd.link);
11 months ago
4
270
 }
271
11 months ago
272
3 months ago
273
 static inline void
274
 Link_AddBefore(
8 months ago
275
 List_Link *link,
276
 List_Link *before
277
 ){
278
 link->prev = before->prev;
279
 link->next = before;
280
 before->prev->next = link;
281
 before->prev = link;
282
 }
283
284
3 months ago
285
 static inline void
286
 List_AddTail(
11 months ago
287
 List_Head *list,
288
 List_Link *link
289
 ){
8 months ago
290
 Link_AddBefore(link, &list->back.link);
11 months ago
4
291
 }
292
11 months ago
293
3 months ago
294
 static inline void
295
 Link_Remove(
11 months ago
296
 List_Link *link
297
 ){
11 months ago
4
298
 link->prev->next = link->next;
299
 link->next->prev = link->prev;
300
 }
301
302
 ///////////////////////////////////////////////////////////////////////////////
303
 // ...2-way linked lists
304
 ///////////////////////////////////////////////////////////////////////////////
305
306
 enum {
307
 Coroutines_Started,
308
 Coroutines_Stopping
309
 };
310
311
 enum {
312
 Chunk_Initial,
8 months ago
313
 Chunk_Split,
11 months ago
4
314
 Chunk_Enter
315
 };
316
11 months ago
317
 typedef enum Coroutine_State {
3 months ago
318
 Coroutine_Free,
319
 Coroutine_Idle,
320
 Coroutine_Running,
321
 Coroutine_Waiting,
322
 Coroutine_Complete
11 months ago
323
 } Coroutine_State;
11 months ago
4
324
325
 enum {
326
 Coroutines_Init,
327
 Coroutines_AllocatedChunk,
328
 Coroutines_CoroutineComplete,
329
 };
330
331
 struct Coroutine {
2 months ago
332
 size_t min_size;
333
 size_t min_headroom;
9 months ago
334
 Coroutines *coroutines; // so can work with it off-thread
335
 List_Link link; // for whichever list it's on
8 months ago
336
 List_Link all_link; // list of all Coroutines
9 months ago
337
 jmp_buf buf; // how to get back to it
2 months ago
338
 unsigned char *base; // where the base of this Coroutine's stack is (= previous block's limit)
339
 unsigned char *limit; // where the limit of this Coroutine's stack is (= next block's base)
9 months ago
340
 unsigned char *guard; // where the stack overrun guard is
341
 Coroutine_Start start; // entry point
342
 void *entry_param; // to pass to start
343
 void *value; // yielded/returned
344
 unsigned char *stack_top; // recorded at yield
Last month
345
 struct Coroutine *chain_root; // The root (entry) of a series of chained coroutines
346
 struct Coroutine *chain_tip; // The tip (yielder) of a series of chained coroutines
11 months ago
347
 Coroutine_State state;
2 months ago
348
349
 int sequence;
11 months ago
4
350
 };
351
352
 struct Coroutines {
10 months ago
353
 _Cor_Mutex mutex;
2 months ago
354
 jmp_buf controller; // to return from Coroutine_Run
11 months ago
355
 jmp_buf chunk_allocated;// for chunk allocation
2 months ago
356
 size_t size_to_retain; // for Chunk_Split
11 months ago
4
357
358
 // singletons
359
 Coroutine *tip; // top of stack chunk
360
 Coroutine *active; // currently running coroutine
2 months ago
361
 Coroutine *primary; // Coroutine_Run coroutine
9 months ago
362
 unsigned char *stack_limit; // when not NULL, where the stack finishes
11 months ago
4
363
2 months ago
364
 Coroutine *spare; // spare struct Coroutine (instead of having to malloc & fail)
365
11 months ago
4
366
 // lists
8 months ago
367
 List_Head all; // all Coroutines (in address order)
368
 List_Head free; // free Coroutines
11 months ago
4
369
 List_Head inactive; // idle or complete
370
 List_Head runable; // running or waiting to run
371
 List_Head waiting; // yielded / waiting to run
10 months ago
372
 _Cor_Mutex waiting_mutex;
11 months ago
4
373
2 months ago
374
 Coroutine *root; // The coroutine for the thread which started Coroutines
375
11 months ago
376
 // Summary of the system
377
 Coroutine_Report report;
378
11 months ago
4
379
 // state
380
 char state;
2 months ago
381
382
 int sequence;
11 months ago
4
383
 };
384
8 months ago
385
 _Cor_thread_local Coroutines *g_c;
7 months ago
386
 _Cor_thread_local unsigned char *g_stack_limit;
11 months ago
4
387
8 months ago
388
 static void ReserveStackSpace(Coroutines *cors, Coroutine *parent, size_t chunk_size, unsigned char *childs_limit);
8 months ago
389
 static void stack_chunk_base(Coroutines *cors, Coroutine *parent, unsigned char *prev_limit, unsigned char *limit);
11 months ago
4
390
11 months ago
391
2 months ago
392
 static size_t
393
 Coroutine_Size
394
 (
395
 Coroutine *cor
396
 ){
397
 if (cor->limit){
398
 return (size_t)StackPointerDiff(cor->limit, cor->base);
399
 } else {
400
 return SIZE_MAX;
401
 }
402
 }
403
404
405
 static size_t
2 months ago
406
 TrimmableSize(
407
 Coroutine *cor
408
 ){
2 months ago
409
 size_t current_used = StackPointerDiff(cor->stack_top, cor->base);
410
 size_t min_size = current_used + cor->min_headroom;
411
 if (min_size < cor->min_size){
412
 min_size = cor->min_size;
2 months ago
413
 }
414
415
 if (cor->limit){
2 months ago
416
 if (Coroutine_Size(cor) < min_size + COROUTINE_MINIMUM_STACK_SIZE){
2 months ago
417
 return 0;
418
 }
419
 }
420
 return min_size;
421
 }
422
423
9 months ago
424
 #define GUARD_PATTERN_SIZE (4)
2 months ago
425
 /// @brief Checks whether a guard pattern is OK
426
 /// @param guard The address of the stack base end of the pattern
427
 /// @return true pattern is ok; false pattern is broken
3 months ago
428
 static inline bool
429
 Guard_Pattern_OK(
11 months ago
430
 unsigned char *guard
431
 ){
9 months ago
432
 return !guard ||
2 months ago
433
 (*StackPointerAdd(guard, 0) == 0xde &&
434
 *StackPointerAdd(guard, 1) == 0xad &&
435
 *StackPointerAdd(guard, 2) == 0xbe &&
436
 *StackPointerAdd(guard, 3) == 0xef);
11 months ago
437
 }
438
439
2 months ago
440
 /// @brief Writes a guard pattern
441
 /// @param guard Where to write the guard pattern (stack base end)
3 months ago
442
 static inline void
443
 Apply_Guard(unsigned char *guard){
2 months ago
444
 *StackPointerAdd(guard, 0) = 0xde;
445
 *StackPointerAdd(guard, 1) = 0xad;
446
 *StackPointerAdd(guard, 2) = 0xbe;
447
 *StackPointerAdd(guard, 3) = 0xef;
9 months ago
448
 }
449
450
6 months ago
451
 #ifndef NDEBUG
2 months ago
452
 /// @brief Checks a list of Coroutines to see whether it's OK
453
 /// @param head The list to check
454
 /// @param state1 One of the states Coroutines in the list are allowed
455
 /// @param state2 One of the states Coroutines in the list are allowed
456
 /// @return Coroutine_OK the is OK; other the list is broken
3 months ago
457
 static Coroutine_Err
458
 CheckListIntegrity(
459
 List_Head *head,
460
 Coroutine_State state1,
461
 Coroutine_State state2
462
 ){
6 months ago
463
 for (List_Link *link = List_Begin(head); Link_NextIsLink(link); link = Link_Next(link)){
464
 Coroutine *candidate = List_Link_Container(Coroutine, link, link);
6 months ago
465
 if (candidate->coroutines != g_c){
466
 return Coroutine_Err_InternalInsistency;
467
 }
468
 if(candidate->state != state1 && candidate->state != state2){
469
 return Coroutine_Err_InternalInsistency;
470
 }
6 months ago
471
 bool found = false;
472
 for (List_Link *link = List_Begin(&g_c->all); Link_NextIsLink(link); link = Link_Next(link)){
473
 Coroutine *candidate2 = List_Link_Container(Coroutine, all_link, link);
474
 if (candidate == candidate2){
475
 found = true;
476
 }
477
 }
6 months ago
478
 if (!found){
479
 return Coroutine_Err_InternalInsistency;
480
 }
6 months ago
481
 }
6 months ago
482
 return Coroutine_OK;
6 months ago
483
 }
484
485
2 months ago
486
 /// @brief Check the integrity of this thread's coroutine system
487
 /// @return Coroutine_OK everything's OK; other something was wrong
3 months ago
488
 static Coroutine_Err
2 months ago
489
 Coroutine_CheckIntegrity_(
3 months ago
490
 void
491
 ){
6 months ago
492
 Coroutine_Err err;
3 months ago
493
 err = CheckListIntegrity(&g_c->free, Coroutine_Free, Coroutine_Free);
6 months ago
494
 if (err){
495
 return err;
496
 }
3 months ago
497
 err = CheckListIntegrity(&g_c->inactive, Coroutine_Idle, Coroutine_Complete);
6 months ago
498
 if (err){
499
 return err;
500
 }
3 months ago
501
 err = CheckListIntegrity(&g_c->runable, Coroutine_Running, Coroutine_Running);
6 months ago
502
 if (err){
503
 return err;
504
 }
3 months ago
505
 err = CheckListIntegrity(&g_c->waiting, Coroutine_Waiting, Coroutine_Waiting);
6 months ago
506
 return err;
6 months ago
507
 }
6 months ago
508
 #endif
509
510
2 months ago
511
 /// @brief Check whether the stack has overrun
512
 /// @return Coroutine_OK the stack hasn't; other it has
3 months ago
513
 static Coroutine_Err
2 months ago
514
 Coroutine_StackHasOverrun(
3 months ago
515
 void
516
 ){
3 months ago
517
 unsigned char *stack_top = (unsigned char *)StackTopNow();
8 months ago
518
 unsigned char *stack_limit = g_c ? g_c->stack_limit : NULL;
2 months ago
519
 if (stack_limit && StackPointerDiff(stack_limit, stack_top) < 0){
9 months ago
520
 // current stack top is beyond limit - we are overrunning NOW
6 months ago
521
 return Coroutine_Err_StackOverrun;
9 months ago
522
 }
8 months ago
523
 Coroutine *me = g_c ? g_c->active : NULL;
9 months ago
524
 if (!me){
6 months ago
525
 return Coroutine_OK;
9 months ago
526
 }
6 months ago
527
 #if COROUTINE_CHECK_INTEGRITY_ON_STACK_CHECK
528
 // Check all coroutines integrity
2 months ago
529
 Coroutine_Err err = Coroutine_CheckIntegrity_();
6 months ago
530
 if (err){
531
 return err;
532
 }
6 months ago
533
 #endif
9 months ago
534
 if (me->guard){
2 months ago
535
 if (StackPointerDiff(me->guard, stack_top) < 0){
2 months ago
536
 // Stack top beyond active stack limit
2 months ago
537
 return Coroutine_Err_StackOverrun;
538
 }
539
 if (!Guard_Pattern_OK(me->guard)){
2 months ago
540
 // Guard pattern trampled
2 months ago
541
 return Coroutine_Err_StackOverrun;
3 months ago
542
 }
9 months ago
543
 }
2 months ago
544
 return Coroutine_OK;
9 months ago
545
 }
546
3 months ago
547
 #ifndef NDEBUG
2 months ago
548
 /// @brief Check system integrity - does stack overrun check and internals check
549
 /// @return Coroutine_OK all is OK; other something was wrong
3 months ago
550
 Coroutine_Err
2 months ago
551
 Coroutine_CheckIntegrity(
3 months ago
552
 void
553
 ){
2 months ago
554
 Coroutine_Err err = Coroutine_StackHasOverrun();
3 months ago
555
 #if !COROUTINE_CHECK_INTEGRITY_ON_STACK_CHECK
556
 if (!err && g_c){
2 months ago
557
 err = Coroutine_CheckIntegrity_();
3 months ago
558
 }
559
 #endif
560
 return err;
561
 }
562
 #endif
9 months ago
563
3 months ago
564
3 months ago
565
 static void
566
 ReserveStackSpace(
8 months ago
567
 Coroutines *cors,
9 months ago
568
 Coroutine *parent,
8 months ago
569
 size_t chunk_size,
570
 unsigned char *childs_limit
11 months ago
571
 ){
9 months ago
572
 unsigned char *chunk_of_stack = alloca(chunk_size);
2 months ago
573
 unsigned char *limit = StackLimitEnd(chunk_of_stack, chunk_of_stack+chunk_size);
574
 unsigned char *guard = StackPointerAdd(limit, -GUARD_PATTERN_SIZE);
9 months ago
575
 #if COROUTINE_RECORD_LOWEST_HEADROOM
576
 for (size_t i = 0; i <= chunk_size-GUARD_PATTERN_SIZE; i += GUARD_PATTERN_SIZE){
5 days ago
577
 Apply_Guard(StackPointerAdd(guard, -(ptrdiff_t)i));
9 months ago
578
 }
579
 #else
2 months ago
580
 Apply_Guard(guard);
9 months ago
581
 #endif
8 months ago
582
 if (parent){
2 months ago
583
 parent->limit = limit;
584
 parent->guard = guard;
8 months ago
585
 }
2 months ago
586
 stack_chunk_base(cors, parent, limit, childs_limit);
11 months ago
4
587
 }
588
11 months ago
589
3 months ago
590
 static void
591
 stack_chunk_base(
8 months ago
592
 Coroutines *cors,
8 months ago
593
 Coroutine *parent,
8 months ago
594
 unsigned char *prev_limit,
595
 unsigned char *limit
11 months ago
596
 ){
2 months ago
597
 MyAssert(cors->spare);
598
 Coroutine *here = cors->spare;
599
 cors->spare = NULL;
600
 here->coroutines = cors;
601
 here->sequence = cors->sequence++;
602
 here->state = Coroutine_Free;
603
 here->base = prev_limit;
604
 here->limit = limit;
605
 here->stack_top = (unsigned char *)StackTopNow();
8 months ago
606
 if (limit){
2 months ago
607
 here->guard = StackPointerAdd(limit, -GUARD_PATTERN_SIZE);
2 months ago
608
 Apply_Guard(here->guard);
2 months ago
609
 } else {
610
 here->guard = NULL;
8 months ago
611
 }
612
8 months ago
613
 // insert into all list
614
 if (parent){
2 months ago
615
 Link_AddAfter(&here->all_link, &parent->all_link);
8 months ago
616
 } else {
2 months ago
617
 List_AddHead(&cors->all, &here->all_link);
8 months ago
618
 }
8 months ago
619
 // add to free list
2 months ago
620
 List_AddTail(&cors->free, &here->link);
8 months ago
621
622
 cors->report.coroutines_pool_size += 1;
623
2 months ago
624
 if (!cors->tip || !Link_NextIsLink(Link_Next(&here->all_link))){
625
 cors->tip = here;
8 months ago
626
 }
627
9 months ago
628
 for(;;){
2 months ago
629
 switch (setjmp(here->buf)) {
9 months ago
630
 case Chunk_Initial:
2 months ago
631
 ready_jmp_buf(here->buf);
632
 if (here->state == Coroutine_Free){
9 months ago
633
 // return to the coroutine allocator
8 months ago
634
 longjmp(cors->chunk_allocated, 1);
9 months ago
635
 } else {
2 months ago
636
 MyAssert(here->state == Coroutine_Complete);
9 months ago
637
 // we finish here to ensure the setjmp is redone
2 months ago
638
 if (cors->primary == here) {
2 months ago
639
 // if primary coroutine - return to Coroutine_Run
8 months ago
640
 longjmp(cors->controller, Coroutines_CoroutineComplete);
9 months ago
641
 }
8 months ago
642
 _Cor_Mutex_Unlock(&cors->mutex);
2 months ago
643
 Coroutine_RunNext();
9 months ago
644
 }
5 months ago
645
 MyAssert(false);
646
 break;
8 months ago
647
 case Chunk_Split:
2 months ago
648
 // Request to split this idle block into two
649
 // g_c->size_to_retain will be set to our shorter size
650
 ReserveStackSpace(here->coroutines, here, g_c->size_to_retain, here->limit);
6 months ago
651
 MyAssert(false);
5 months ago
652
 break;
9 months ago
653
 case Chunk_Enter:
654
 // request to start a coroutine (ie use the chunk for a coroutine)
655
 // arrive here with mutex locked
2 months ago
656
 MyAssert(here->state == Coroutine_Running);
657
 here->coroutines->active = here;
2 months ago
658
 EnlargeActiveIfPossible();
8 months ago
659
 _Cor_Mutex_Unlock(&cors->mutex);
2 months ago
660
 here->value = here->start(here->entry_param);
11 months ago
661
9 months ago
662
 // check the guard
2 months ago
663
 MyAssert(Guard_Pattern_OK(here->guard));
11 months ago
664
2 months ago
665
 here->stack_top = (unsigned char *)StackTopNow();
666
 _Cor_Mutex_Lock(&here->coroutines->mutex);
2 months ago
667
 TrimActiveIfPossible();
2 months ago
668
 here->coroutines->active = NULL;
669
 MyAssert(here->state == Coroutine_Running);
670
 Link_Remove(&here->link);
671
 here->state = Coroutine_Complete;
672
 List_AddTail(&here->coroutines->inactive, &here->link);
9 months ago
673
 // Coroutine has completed
674
 // Loop round to redo the setjmp() - if this coroutine yielded, then the setjmp will
675
 // need reseting
5 months ago
676
 break;
11 months ago
4
677
 }
678
 }
679
 }
680
11 months ago
681
3 months ago
682
 static void
2 months ago
683
 Coroutine_RunNext(
3 months ago
684
 void
685
 ){
11 months ago
686
 // arrive here with mutex unlocked
8 months ago
687
 _Cor_Mutex_Lock(&g_c->waiting_mutex);
688
 _Cor_Mutex_Lock(&g_c->mutex);
689
 Coroutine *next = List_Link_Container(Coroutine, link, List_GetHead(&g_c->runable));
3 months ago
690
 MyAssert(next->state == Coroutine_Running);
11 months ago
691
 longjmp(next->buf, Chunk_Enter);
6 months ago
692
 MyAssert(false);
11 months ago
693
 }
694
695
2 months ago
696
 /// @brief Ensures there's a spare Coroutine for cors
697
 /// @param cors The Coroutines to ensure a spare for
698
 /// @return true there is a spare; false there isn't
699
 static inline bool
700
 EnsureSpare
701
 (
702
 Coroutines *cors
703
 ){
704
 if (cors->spare){
705
 return true;
706
 }
707
 cors->spare = malloc(sizeof(*cors->spare));
708
 return cors->spare != NULL;
709
 }
710
711
3 months ago
712
 static Coroutine_Err
713
 Coroutines_ctor(
2 months ago
714
 Coroutines *cors,
715
 size_t min_size,
716
 size_t min_headroom
3 months ago
717
 ){
6 months ago
718
 if (_Cor_Mutex_ctor(&cors->mutex)){
2 months ago
719
 goto error;
6 months ago
720
 }
8 months ago
721
 cors->primary = NULL;
7 months ago
722
 cors->stack_limit = g_stack_limit;
11 months ago
4
723
8 months ago
724
 List_Init(&cors->all);
8 months ago
725
 List_Init(&cors->free);
726
 List_Init(&cors->inactive);
727
 List_Init(&cors->runable);
728
 List_Init(&cors->waiting);
6 months ago
729
 if (_Cor_Mutex_ctor(&cors->waiting_mutex)){
2 months ago
730
 goto error1;
6 months ago
731
 }
732
 if (_Cor_Mutex_Lock(&cors->waiting_mutex)){
2 months ago
733
 goto error2;
6 months ago
734
 }
11 months ago
4
735
8 months ago
736
 cors->report.coroutines_created = 0;
737
 cors->report.coroutines_pool_size = 0;
738
 cors->report.largest_stack = 0;
2 months ago
739
 cors->spare = NULL;
740
 cors->sequence = 0;
8 months ago
741
2 months ago
742
 Coroutine *cor = malloc(sizeof(*cor));
743
 cor->min_size = min_size;
744
 cor->min_headroom = min_headroom;
745
 cor->coroutines = cors;
746
 cor->sequence = cors->sequence++;
747
 cor->state = Coroutine_Running;
748
 cor->base = (unsigned char *)&cor;
Last month
749
 cor->chain_root = cor;
750
 cor->chain_tip = cor;
2 months ago
751
 if (cors->stack_limit){
752
 cor->limit = cors->stack_limit;
753
 cor->guard = StackPointerAdd(cor->limit, -GUARD_PATTERN_SIZE);
754
 Apply_Guard(cor->guard);
755
 } else {
756
 cor->limit = cor->guard = NULL;
11 months ago
4
757
 }
2 months ago
758
 cors->root = cor;
759
 List_AddHead(&cors->all, &cor->all_link);
760
 List_AddTail(&cors->runable, &cor->link);
761
 cors->tip = cors->active = cor;
11 months ago
762
2 months ago
763
 cors->report.coroutines_pool_size += 1;
8 months ago
764
765
 cors->state = Coroutines_Started;
6 months ago
766
 return Coroutine_OK;
2 months ago
767
 error2:
768
 _Cor_Mutex_dtor(&cors->waiting_mutex);
769
 error1:
770
 _Cor_Mutex_dtor(&cors->mutex);
771
 error:
772
 return Coroutine_Err_CouldNotInitialiseSystem;
6 months ago
773
 }
8 months ago
774
3 months ago
775
 static void
776
 Coroutines_dtor(
777
 Coroutines *cors
778
 ){
6 months ago
779
 _Cor_Mutex_Lock(&cors->mutex);
780
 cors->state = Coroutines_Stopping;
781
6 months ago
782
 MyAssert(List_IsEmpty(&cors->inactive));
2 months ago
783
784
 // free the spare
785
 if (cors->spare){
786
 free(cors->spare);
787
 }
788
789
 // free all non-spares
790
 List_Link *link;
791
 List_Link *next;
792
 for (link = List_Begin(&cors->all); Link_NextIsLink(link); link = next){
793
 next = Link_Next(link);
794
 Coroutine *cor = List_Link_Container(Coroutine, all_link, link) ;
795
 free(cor);
796
 }
797
6 months ago
798
 _Cor_Mutex_Unlock(&cors->waiting_mutex);
799
 _Cor_Mutex_dtor(&cors->waiting_mutex);
800
6 months ago
801
 MyAssert(cors->state == Coroutines_Stopping);
6 months ago
802
 _Cor_Mutex_Unlock(&cors->mutex);
803
 _Cor_Mutex_dtor(&cors->mutex);
11 months ago
4
804
 }
805
11 months ago
806
3 months ago
807
 Coroutine_Err
2 months ago
808
 Coroutine_RunSystem(
2 months ago
809
 size_t min_size,
810
 size_t min_headroom,
3 months ago
811
 Coroutine_SystemStart start,
812
 void *value
813
 ){
6 months ago
814
 CHECK_SYSTEM_NOT_RUNNING
815
816
 Coroutines cors;
2 months ago
817
 Coroutine_Err err = Coroutines_ctor(&cors, min_size, min_headroom);
6 months ago
818
 if (err){
819
 return err;
820
 }
821
 g_c = &cors;
2 months ago
822
 err = start(value, cors.root);
6 months ago
823
 g_c = NULL;
824
 Coroutines_dtor(&cors);
825
 return err;
826
 }
827
828
3 months ago
829
 void
2 months ago
830
 Coroutine_SetStackLimit(
3 months ago
831
 void *limit
832
 ){
2 months ago
833
 MyAssert(!limit || !g_c || !g_stack_limit || StackPointerDiff((unsigned char *)limit, (unsigned char *)g_stack_limit) <= 0);
7 months ago
834
 g_stack_limit = limit;
835
 if (g_c){
836
 g_c->stack_limit = limit;
2 months ago
837
 if (g_c->tip){
838
 g_c->tip->limit = limit;
839
 g_c->tip->guard = StackPointerAdd(limit, -GUARD_PATTERN_SIZE);
840
 Apply_Guard(g_c->tip->guard);
841
 }
7 months ago
842
 }
9 months ago
843
 }
844
845
6 months ago
846
 #if COROUTINE_RECORD_LOWEST_HEADROOM
3 months ago
847
 static size_t
2 months ago
848
 Coroutine_UpdateMinimumHeadroom(
3 months ago
849
 List_Head *list,
850
 size_t headroom
851
 ){
6 months ago
852
 for (List_Link *link = List_Begin(list); Link_NextIsLink(link); link = Link_Next(link)){
11 months ago
853
 Coroutine *cor = List_Link_Container(Coroutine, link, link);
9 months ago
854
 if (cor->guard){
2 months ago
855
 size_t chunk_size = Coroutine_Size(cor);
856
 for (size_t i = 0; i <= chunk_size-GUARD_PATTERN_SIZE; i += GUARD_PATTERN_SIZE){
5 days ago
857
 if (!Guard_Pattern_OK(StackPointerAdd(cor->guard, -(ptrdiff_t)i))){
6 months ago
858
 headroom = i < headroom ? i : headroom;
9 months ago
859
 break;
860
 }
11 months ago
861
 }
862
 }
863
 }
6 months ago
864
 return headroom;
865
 }
866
 #endif
867
868
3 months ago
869
 Coroutine_Report
2 months ago
870
 Coroutine_GetReport(
3 months ago
871
 void
872
 ){
6 months ago
873
 if (g_c){
874
 size_t headroom;
6 months ago
875
 #if COROUTINE_RECORD_LOWEST_HEADROOM
6 months ago
876
 _Cor_Mutex_Lock(&g_c->mutex);
877
 headroom = g_c->report.lowest_headroom;
2 months ago
878
 headroom = Coroutine_UpdateMinimumHeadroom(&g_c->inactive, headroom);
879
 headroom = Coroutine_UpdateMinimumHeadroom(&g_c->runable, headroom);
880
 headroom = Coroutine_UpdateMinimumHeadroom(&g_c->waiting, headroom);
6 months ago
881
 _Cor_Mutex_Unlock(&g_c->mutex);
9 months ago
882
 #else
6 months ago
883
 headroom = 0;
9 months ago
884
 #endif
6 months ago
885
 g_c->report.lowest_headroom = headroom;
11 months ago
886
6 months ago
887
 return g_c->report;
888
 } else {
889
 Coroutine_Report ret = {0, 0, 0, 0};
890
 return ret;
891
 }
6 months ago
892
 }
893
894
3 months ago
895
 struct Coroutine_Run_Params {
6 months ago
896
 Coroutine_Start start;
897
 void *value;
898
 void **result;
899
 };
900
3 months ago
901
 static Coroutine_Err
2 months ago
902
 Coroutine_Run_Starter(
2 months ago
903
 void *_params,
904
 Coroutine *root
3 months ago
905
 ){
2 months ago
906
 (void)root;
3 months ago
907
 struct Coroutine_Run_Params *params = (struct Coroutine_Run_Params *)_params;
6 months ago
908
2 months ago
909
 void *res = params->start(params->value);
910
 if (params->result){
911
 *params->result = res;
6 months ago
912
 }
2 months ago
913
914
 return Coroutine_OK;
6 months ago
915
 }
916
917
2 months ago
918
 Coroutine_Err Coroutine_Run(
2 months ago
919
 size_t min_size,
920
 size_t min_headroom,
11 months ago
921
 Coroutine_Start start,
8 months ago
922
 void *value,
923
 void **result
11 months ago
924
 ){
6 months ago
925
 if (!g_c){
2 months ago
926
 struct Coroutine_Run_Params params = {start, value, result};
2 months ago
927
 return Coroutine_RunSystem(min_size, min_headroom, Coroutine_Run_Starter, &params);
6 months ago
928
 }
929
930
 // We are in an active coroutine, so call start() directly
931
 CHECK_STACK_OVERRUN
932
 void *res = start(value);
8 months ago
933
 if (result){
6 months ago
934
 *result = res;
8 months ago
935
 }
6 months ago
936
8 months ago
937
 // no failures, so...
6 months ago
938
 return Coroutine_OK;
11 months ago
4
939
 }
940
941
2 months ago
942
 static Coroutine *Coroutine_New_Lock_Assumed(
2 months ago
943
 size_t min_size,
944
 size_t min_headroom,
8 months ago
945
 Coroutine_Start start
946
 ){
947
 List_Link *link;
948
2 months ago
949
 if (!EnsureSpare(g_c)){
950
 return NULL;
8 months ago
951
 }
952
8 months ago
953
 Coroutine *cor = NULL;
8 months ago
954
 for (link = List_Begin(&g_c->free); Link_NextIsLink(link); link = Link_Next(link)){
8 months ago
955
 Coroutine *candidate = List_Link_Container(Coroutine, link, link);
6 months ago
956
 MyAssert(candidate->coroutines == g_c);
2 months ago
957
 MyAssert(candidate->state == Coroutine_Free);
958
 if (candidate->limit){
959
 size_t candidate_size = Coroutine_Size(candidate);
960
 if (candidate_size < min_size){
961
 continue;
9 months ago
962
 }
2 months ago
963
 }
964
 // candidate has no limit, or is big enough
8 months ago
965
2 months ago
966
 // check if it's 'better' (lower in the C stack) than cor
967
 if (!cor || StackPointerDiff(candidate->base, cor->base) < 0){
8 months ago
968
 // chunk big enough, and a better choice than cor
969
 cor = candidate;
970
 }
971
 }
972
2 months ago
973
 if (!cor){
974
 return NULL;
975
 }
976
2 months ago
977
 // use this free block
2 months ago
978
 cor->min_size = min_size;
979
 cor->min_headroom = min_headroom;
980
 cor->state = Coroutine_Idle;
981
 cor->start = start;
982
 cor->value = NULL;
Last month
983
 cor->chain_root = cor;
984
 cor->chain_tip = cor;
2 months ago
985
 Link_Remove(&cor->link);
986
 List_AddHead(&g_c->inactive, &cor->link);
987
988
 // trim down to an appropriate size
989
 size_t trimmable_size = TrimmableSize(cor);
990
 if (trimmable_size){
991
 // split block into two
992
 if (EnsureSpare(g_c)){
993
 g_c->size_to_retain = trimmable_size;
8 months ago
994
 if (!setjmp(g_c->chunk_allocated)){
3 months ago
995
 ready_jmp_buf(g_c->chunk_allocated);
8 months ago
996
 longjmp(cor->buf, Chunk_Split);
8 months ago
997
 }
998
 }
11 months ago
4
999
 }
1000
2 months ago
1001
 g_c->report.coroutines_created += 1;
8 months ago
1002
2 months ago
1003
 return cor;
1004
 }
1005
1006
1007
 static void
1008
 TrimActiveIfPossible
1009
 (
1010
 void
1011
 ){
1012
 MyAssert(g_c);
1013
1014
 // If we're the active coroutine, free the tail
1015
 Coroutine *active = g_c->active;
1016
 MyAssert(active);
1017
1018
 active->stack_top = (unsigned char *)StackTopNow();
1019
 ptrdiff_t timmablesize = TrimmableSize(active);
1020
 if (!timmablesize){
1021
 return;
8 months ago
1022
 }
2 months ago
1023
1024
 if (!EnsureSpare(g_c)){
1025
 return;
8 months ago
1026
 }
1027
2 months ago
1028
 // enough space for a second coroutine so free the unused stack space from active
1029
 if (!setjmp(g_c->chunk_allocated)){
1030
 ready_jmp_buf(g_c->chunk_allocated);
1031
 ReserveStackSpace(g_c, active, timmablesize - StackPointerDiff(active->stack_top, active->base), active->limit);
1032
 MyAssert(false);
1033
 }
1034
 }
1035
1036
1037
 static void
1038
 EnlargeActiveIfPossible(
1039
 void
1040
 ){
1041
 MyAssert(g_c);
1042
 Coroutine *active = g_c->active;
1043
 MyAssert(active);
1044
1045
 List_Link *next = Link_Next(&active->all_link);
1046
 if (!Link_NextIsLink(next)){
1047
 return;
1048
 }
1049
 Coroutine *cor = List_Link_Container(Coroutine, all_link, next);
1050
 if (cor->state != Coroutine_Free){
1051
 return;
1052
 }
1053
1054
 // Following block is free, merge with this
1055
 active->limit = cor->limit;
1056
 active->guard = cor->guard;
8 months ago
1057
 Link_Remove(&cor->link);
2 months ago
1058
 Link_Remove(&cor->all_link);
1059
 if (g_c->tip == cor){
1060
 g_c->tip = active;
1061
 }
1062
 if (g_c->spare){
1063
 free(cor);
1064
 } else {
1065
 g_c->spare = cor;
1066
 }
8 months ago
1067
 }
11 months ago
1068
8 months ago
1069
3 months ago
1070
 Coroutine *
2 months ago
1071
 Coroutine_New(
2 months ago
1072
 size_t min_size,
1073
 size_t min_headroom,
8 months ago
1074
 Coroutine_Start start
1075
 ){
2 months ago
1076
 MyAssert(g_c && g_c->state == Coroutines_Started);
2 months ago
1077
 MyAssert(!Coroutine_StackHasOverrun());
8 months ago
1078
2 months ago
1079
 // Make the paramaters make sense
1080
 if (min_size < min_headroom){
1081
 min_size = min_headroom;
1082
 }
1083
8 months ago
1084
 _Cor_Mutex_Lock(&g_c->mutex);
1085
2 months ago
1086
 TrimActiveIfPossible();
8 months ago
1087
2 months ago
1088
 Coroutine *cor = Coroutine_New_Lock_Assumed(min_size, min_headroom, start);
2 months ago
1089
1090
 if (cor && Coroutine_Size(cor) > g_c->report.largest_stack){
1091
 g_c->report.largest_stack = Coroutine_Size(cor);
8 months ago
1092
 }
1093
2 months ago
1094
 EnlargeActiveIfPossible();
1095
8 months ago
1096
 _Cor_Mutex_Unlock(&g_c->mutex);
1097
11 months ago
4
1098
 return cor;
1099
 }
1100
11 months ago
1101
3 months ago
1102
 void
2 months ago
1103
 Coroutine_Delete(
11 months ago
1104
 Coroutine *cor
1105
 ){
2 months ago
1106
 MyAssert(!Coroutine_StackHasOverrun());
8 months ago
1107
 if (cor){
1108
 Coroutines *cors = cor->coroutines;
1109
 _Cor_Mutex_Lock(&cors->mutex);
3 months ago
1110
 MyAssert(cor->state == Coroutine_Idle || cor->state == Coroutine_Complete);
6 months ago
1111
1112
 #if COROUTINE_RECORD_LOWEST_HEADROOM
1113
 if (cor->guard){
2 months ago
1114
 unsigned char *guard = cor->guard;
1115
 ptrdiff_t chunk_size = Coroutine_Size(cor);
1116
 ptrdiff_t myheadroom;
1117
 for (myheadroom = 0; myheadroom <= chunk_size-(ptrdiff_t)GUARD_PATTERN_SIZE; myheadroom += GUARD_PATTERN_SIZE){
1118
 if (!Guard_Pattern_OK(StackPointerAdd(guard, -myheadroom))){
6 months ago
1119
 break;
1120
 }
1121
 }
2 months ago
1122
 if (myheadroom < (ptrdiff_t)g_c->report.lowest_headroom || g_c->report.lowest_headroom == 0){
6 months ago
1123
 g_c->report.lowest_headroom = myheadroom;
1124
 }
1125
 }
1126
 #endif
1127
3 months ago
1128
 cor->state = Coroutine_Free;
8 months ago
1129
 Link_Remove(&cor->link);
1130
8 months ago
1131
 // insert into free list
1132
 List_AddHead(&cors->free, &cor->link);
8 months ago
1133
1134
 // Check for merge with following Coroutine
8 months ago
1135
 List_Link *link = Link_Next(&cor->all_link);
8 months ago
1136
 if (Link_NextIsLink(link)){
8 months ago
1137
 Coroutine *listcor = List_Link_Container(Coroutine, all_link, link);
3 months ago
1138
 if (listcor->state == Coroutine_Free){
8 months ago
1139
 // merge
1140
 cor->limit = listcor->limit;
1141
 cor->guard = listcor->guard;
8 months ago
1142
 Link_Remove(&listcor->all_link);
8 months ago
1143
 Link_Remove(&listcor->link);
8 months ago
1144
 if (g_c->tip == listcor){
1145
 g_c->tip = cor;
1146
 }
2 months ago
1147
1148
 // free up the now unused struct
1149
 if (g_c->spare){
1150
 free(listcor);
1151
 } else {
1152
 g_c->spare = listcor;
1153
 }
8 months ago
1154
 }
1155
 }
1156
1157
 // check for merge with prev coroutine
8 months ago
1158
 link = Link_Prev(&cor->all_link);
8 months ago
1159
 if (Link_PrevIsLink(link)){
8 months ago
1160
 Coroutine *listcor = List_Link_Container(Coroutine, all_link, link);
3 months ago
1161
 if (listcor->state == Coroutine_Free){
8 months ago
1162
 // merge
1163
 listcor->limit = cor->limit;
1164
 listcor->guard = cor->guard;
8 months ago
1165
 Link_Remove(&cor->all_link);
8 months ago
1166
 Link_Remove(&cor->link);
8 months ago
1167
 if (g_c->tip == cor){
1168
 g_c->tip = listcor;
1169
 }
2 months ago
1170
1171
 // free up the now unused struct
1172
 if (g_c->spare){
1173
 free(cor);
1174
 } else {
1175
 g_c->spare = cor;
1176
 }
8 months ago
1177
 }
1178
 }
2 months ago
1179
1180
 EnlargeActiveIfPossible();
8 months ago
1181
8 months ago
1182
 _Cor_Mutex_Unlock(&cors->mutex);
1183
 }
11 months ago
4
1184
 }
1185
11 months ago
1186
2 months ago
1187
 // Coroutine_Continue, assuming the mutex is claimed
6 months ago
1188
 // return false for success, true for something went wrong
3 months ago
1189
 static Coroutine_Err
2 months ago
1190
 Coroutine_Continue_(
6 months ago
1191
 Coroutines *cors,
11 months ago
1192
 Coroutine *cor,
1193
 void *value,
1194
 bool early
1195
 ){
Last month
1196
 cor = cor->chain_tip;
3 months ago
1197
 if (cor->state == Coroutine_Running){
6 months ago
1198
 // already running
6 months ago
1199
 return Coroutine_OK;
6 months ago
1200
 }
3 months ago
1201
 if (cor->state != Coroutine_Idle && cor->state != Coroutine_Waiting){
6 months ago
1202
 return Coroutine_Err_WrongState;
1203
 }
11 months ago
4
1204
 cor->entry_param = value;
3 months ago
1205
 cor->state = Coroutine_Running;
8 months ago
1206
 Link_Remove(&cor->link);
11 months ago
4
1207
 if ( early ) {
11 months ago
1208
 List_AddHead(&cors->runable, &cor->link);
11 months ago
4
1209
 } else {
11 months ago
1210
 List_AddTail(&cors->runable, &cor->link);
11 months ago
4
1211
 }
10 months ago
1212
 _Cor_Mutex_Unlock(&cors->waiting_mutex);
6 months ago
1213
 return Coroutine_OK;
10 months ago
1214
 }
1215
1216
3 months ago
1217
 Coroutine_Err
2 months ago
1218
 Coroutine_Continue(
10 months ago
1219
 Coroutine *cor,
1220
 void *value,
1221
 bool early
1222
 ){
2 months ago
1223
 MyAssert(!Coroutine_StackHasOverrun());
10 months ago
1224
 Coroutines *cors = cor->coroutines;
10 months ago
1225
 _Cor_Mutex_Lock(&cors->mutex);
2 months ago
1226
 Coroutine_Err err = Coroutine_Continue_(cors, cor, value, early);
10 months ago
1227
 _Cor_Mutex_Unlock(&cors->mutex);
6 months ago
1228
 return err;
11 months ago
4
1229
 }
1230
11 months ago
1231
3 months ago
1232
 void *
2 months ago
1233
 Coroutine_Yield(
11 months ago
1234
 void *value,
3 months ago
1235
 Coroutine_YieldCallback on_yield,
11 months ago
1236
 void *yield_me
11 months ago
1237
 ){
6 months ago
1238
 MyAssert(g_c);
8 months ago
1239
 Coroutine *me = g_c->active;
6 months ago
1240
 MyAssert(me);
2 months ago
1241
 MyAssert(!Coroutine_StackHasOverrun());
11 months ago
1242
8 months ago
1243
 _Cor_Mutex_Lock(&g_c->mutex);
11 months ago
1244
 Coroutines *cors = me->coroutines;
3 months ago
1245
 MyAssert(me && me->state == Coroutine_Running && cors == g_c);
3 months ago
1246
 me->stack_top = (unsigned char *)StackTopNow();
Last month
1247
 me->value = NULL;
1248
 me->chain_root->value = value;
5 days ago
1249
 Coroutine *prev_tip = me->chain_root->chain_tip;
Last month
1250
 me->chain_root->chain_tip = me;
3 months ago
1251
 me->state = Coroutine_Waiting;
11 months ago
1252
2 months ago
1253
 TrimActiveIfPossible();
1254
8 months ago
1255
 Link_Remove(&me->link);
10 months ago
1256
 if (!List_IsEmpty(&cors->runable)){
10 months ago
1257
 _Cor_Mutex_Unlock(&cors->waiting_mutex);
10 months ago
1258
 }
11 months ago
1259
 List_AddTail(&cors->waiting, &me->link);
11 months ago
1260
11 months ago
1261
 switch (setjmp(me->buf)){
1262
 case Chunk_Initial:
3 months ago
1263
 ready_jmp_buf(me->buf);
10 months ago
1264
 _Cor_Mutex_Unlock(&cors->mutex);
11 months ago
1265
 on_yield(yield_me);
2 months ago
1266
 Coroutine_RunNext();
6 months ago
1267
 MyAssert(false);
5 months ago
1268
 break;
11 months ago
1269
 case Chunk_Enter:
1270
 // arrive here with mutex locked
11 months ago
1271
 cors->active = me;
2 months ago
1272
 MyAssert(!Coroutine_StackHasOverrun());
11 months ago
1273
 // when we return here - we are running again
3 months ago
1274
 MyAssert(me->state == Coroutine_Running);
2 months ago
1275
 EnlargeActiveIfPossible();
5 days ago
1276
 me->chain_root->chain_tip = prev_tip;
11 months ago
1277
 void *res = me->entry_param;
10 months ago
1278
 _Cor_Mutex_Unlock(&cors->mutex);
11 months ago
1279
 return res;
11 months ago
4
1280
 }
5 months ago
1281
 MyAssert(false);
11 months ago
1282
 return NULL;
11 months ago
4
1283
 }
1284
11 months ago
1285
3 months ago
1286
 void *
2 months ago
1287
 Coroutine_GetValue(
11 months ago
1288
 Coroutine *cor
1289
 ){
11 months ago
4
1290
 return cor->value;
1291
 }
1292
11 months ago
1293
3 months ago
1294
 Coroutine *
2 months ago
1295
 Coroutine_GetActive(
3 months ago
1296
 void
1297
 ){
Last month
1298
 return g_c ? g_c->active->chain_root : NULL;
11 months ago
4
1299
 }
1300
11 months ago
1301
2 months ago
1302
 ptrdiff_t
2 months ago
1303
 Coroutine_GetStackHeadroom(
3 months ago
1304
 void
1305
 ){
6 months ago
1306
 Coroutine *me = g_c ? g_c->active : NULL;
9 months ago
1307
 if (!me){
1308
 // no active coroutine
6 months ago
1309
 if (g_stack_limit){
2 months ago
1310
 return StackPointerDiff(g_stack_limit, (unsigned char *)StackTopNow());
9 months ago
1311
 } else {
1312
 // no information where the stack ends - return something
2 months ago
1313
 // The biggest ptrdiff_t possible
1314
 return PTRDIFF_MAX;
9 months ago
1315
 }
2 months ago
1316
 } else {
1317
 if (me->guard){
1318
 return StackPointerDiff(me->guard, (unsigned char *)StackTopNow());
1319
 } else {
2 months ago
1320
 // The biggest ptrdiff_t possible
1321
 return PTRDIFF_MAX;
2 months ago
1322
 }
9 months ago
1323
 }
11 months ago
1324
 }
1325
1326
3 months ago
1327
 void *
2 months ago
1328
 Coroutine_GetStackHWM(
3 months ago
1329
 void
1330
 ){
6 months ago
1331
 MyAssert(g_c);
2 months ago
1332
 MyAssert(g_c->state == Coroutines_Started);
2 months ago
1333
 MyAssert(!Coroutine_StackHasOverrun());
9 months ago
1334
 // Find where the guards end
2 months ago
1335
 unsigned char *guard = g_c->active->guard;
1336
 if (guard){
2 months ago
1337
 ptrdiff_t headroom = Coroutine_GetStackHeadroom();
2 months ago
1338
 for (ptrdiff_t i = 0; i <= headroom; i += GUARD_PATTERN_SIZE){
1339
 if (!Guard_Pattern_OK(StackPointerAdd(guard, -i))){
1340
 return StackPointerAdd(guard, i);
1341
 }
1342
 }
9 months ago
1343
 }
1344
 return guard;
1345
 }
10 months ago
1346
1347
3 months ago
1348
 void
2 months ago
1349
 Coroutine_ClearStackForHWM(
3 months ago
1350
 void
1351
 ){
6 months ago
1352
 MyAssert(g_c);
2 months ago
1353
 MyAssert(!Coroutine_StackHasOverrun());
2 months ago
1354
 unsigned char *guard = g_c->active->guard;
1355
 if (guard){
2 months ago
1356
 ptrdiff_t headroom = Coroutine_GetStackHeadroom();
2 months ago
1357
 for (ptrdiff_t i = 0; i <= headroom; i += GUARD_PATTERN_SIZE){
1358
 Apply_Guard(StackPointerAdd(guard, -i));
1359
 }
9 months ago
1360
 }
1361
 }
1362
1363
3 months ago
1364
 static bool
2 months ago
1365
 Coroutine_CanStartCoroutine_Lock_Assumed(
2 months ago
1366
 size_t min_size
8 months ago
1367
 ){
1368
 if (!g_c->stack_limit){
9 months ago
1369
 return true;
1370
 }
9 months ago
1371
2 months ago
1372
 // check free list
8 months ago
1373
 List_Link *link;
1374
 for (link = List_Begin(&g_c->free); Link_NextIsLink(link); link = Link_Next(link)){
1375
 Coroutine *cor = List_Link_Container(Coroutine, link, link);
2 months ago
1376
 size_t cor_size = Coroutine_Size(cor);
1377
 if (cor_size >= min_size){
8 months ago
1378
 return true;
1379
 }
1380
 }
1381
2 months ago
1382
 // Check if this coroutine has enough size
1383
 Coroutine *me = g_c->active;
1384
 me->stack_top = (unsigned char *)StackTopNow();
1385
 ptrdiff_t reduced_size = TrimmableSize(g_c->active);
1386
 if (reduced_size && StackPointerDiff(g_c->active->limit, g_c->active->base) - reduced_size > (ptrdiff_t)min_size){
1387
 return true;
1388
 }
1389
8 months ago
1390
 return false;
9 months ago
1391
 }
1392
3 months ago
1393
 bool
2 months ago
1394
 Coroutine_CanStartCoroutine(
8 months ago
1395
 size_t size
1396
 ){
2 months ago
1397
 MyAssert(g_c && g_c->state == Coroutines_Started);
2 months ago
1398
 MyAssert(!Coroutine_StackHasOverrun());
6 months ago
1399
8 months ago
1400
 _Cor_Mutex_Lock(&g_c->mutex);
1401
2 months ago
1402
 bool result = Coroutine_CanStartCoroutine_Lock_Assumed(size);
8 months ago
1403
1404
 _Cor_Mutex_Unlock(&g_c->mutex);
1405
1406
 return result;
1407
 }
1408
2 months ago
1409
 static bool
1410
 Coroutine_GetUsefulFreeSpace_Lock_Assumed(
2 months ago
1411
 size_t min_size,
2 months ago
1412
 size_t overhead
1413
 ){
1414
 if (!g_c->stack_limit){
1415
 return SIZE_MAX;
1416
 }
1417
1418
 size_t total = 0;
1419
1420
 // check free list
1421
 List_Link *link;
1422
 for (link = List_Begin(&g_c->free); Link_NextIsLink(link); link = Link_Next(link)){
1423
 Coroutine *cor = List_Link_Container(Coroutine, link, link);
1424
 size_t cor_size = Coroutine_Size(cor);
2 months ago
1425
 if (cor_size >= min_size){
2 months ago
1426
 total += cor_size - overhead;
1427
 }
1428
 }
1429
1430
 return total;
1431
 }
1432
1433
 size_t
1434
 Coroutine_GetUsefulFreeSpace(
2 months ago
1435
 size_t min_size,
2 months ago
1436
 size_t overhead
1437
 ){
1438
 MyAssert(g_c && g_c->state == Coroutines_Started);
1439
 MyAssert(!Coroutine_StackHasOverrun());
1440
2 months ago
1441
 if (min_size < overhead){
1442
 min_size = overhead;
1443
 }
1444
2 months ago
1445
 _Cor_Mutex_Lock(&g_c->mutex);
1446
2 months ago
1447
 size_t result = Coroutine_GetUsefulFreeSpace_Lock_Assumed(min_size, overhead);
2 months ago
1448
1449
 _Cor_Mutex_Unlock(&g_c->mutex);
1450
1451
 return result;
1452
 }
1453
3 months ago
1454
 void *
2 months ago
1455
 Coroutine_GetCStackTop(
3 months ago
1456
 void
1457
 ){
2 months ago
1458
 if (!g_c || g_c->tip == g_c->active){
1459
 return (void *)StackTopNow();
9 months ago
1460
 }
2 months ago
1461
1462
 return g_c->tip->stack_top;
11 months ago
1463
 }
1464
1465
3 months ago
1466
 // Inspired by cpython...
1467
 #ifdef __has_builtin
1468
 # define Coroutine__has_builtin(x) __has_builtin(x)
1469
 #else
1470
 # define Coroutine__has_builtin(x) 0
1471
 #endif
1472
1473
 #if !Coroutine__has_builtin(__builtin_frame_address) && !defined(__GNUC__) && !defined(_MSC_VER)
1474
 static uintptr_t return_pointer_as_int(char* p) {
1475
 return (uintptr_t)p;
9 months ago
1476
 }
3 months ago
1477
 #endif
9 months ago
1478
3 months ago
1479
 static inline uintptr_t
1480
 StackTopNow(void) {
1481
 #if Coroutine__has_builtin(__builtin_frame_address) || defined(__GNUC__)
1482
 return (uintptr_t)__builtin_frame_address(0);
1483
 #elif defined(_MSC_VER)
1484
 return (uintptr_t)_AddressOfReturnAddress();
1485
 #else
1486
 char here;
1487
 /* Avoid compiler warning about returning stack address */
1488
 return return_pointer_as_int(&here);
1489
 #endif
1490
 }
1491
 // ...inspired by cpython
9 months ago
1492
3 months ago
1493
3 months ago
1494
 struct Coroutine_ChainParam {
11 months ago
1495
 Coroutine_Start start;
1496
 void *value;
1497
 Coroutine *ret;
1498
 };
1499
1500
3 months ago
1501
 static void *
2 months ago
1502
 Coroutine_ChainFn(
11 months ago
1503
 void *param
1504
 ){
Last month
1505
 struct Coroutine_ChainParam params = *(struct Coroutine_ChainParam *)param;
1506
 void *res = params.start(params.value);
1507
 params.ret->chain_tip = params.ret;
1508
 return (void *)(uintptr_t)Coroutine_Continue(params.ret, res, true);
11 months ago
1509
 }
1510
1511
3 months ago
1512
 static void
2 months ago
1513
 Coroutine_ChainYield(
11 months ago
1514
 void *unused
1515
 ){
1516
 (void)unused;
1517
 }
1518
1519
3 months ago
1520
 Coroutine_Err
2 months ago
1521
 Coroutine_Chain(
2 months ago
1522
 size_t min_size,
1523
 size_t min_headroom,
11 months ago
1524
 Coroutine_Start start,
8 months ago
1525
 void *value,
1526
 void **result
11 months ago
1527
 ){
2 months ago
1528
 MyAssert(!Coroutine_StackHasOverrun());
1529
 Coroutine *cor = Coroutine_New(min_size, min_headroom, Coroutine_ChainFn);
8 months ago
1530
 if (!cor){
1531
 // failed
6 months ago
1532
 return Coroutine_Err_NoStack;
8 months ago
1533
 }
Last month
1534
 cor->chain_root = g_c->active->chain_root;
1535
 g_c->active->chain_tip = cor;
3 months ago
1536
 struct Coroutine_ChainParam params = {
11 months ago
1537
 start,
1538
 value,
Last month
1539
 g_c->active,
11 months ago
1540
 };
2 months ago
1541
 Coroutine_Err err = Coroutine_Continue(cor, &params, true);
2 months ago
1542
 if (!err){
1543
 void *res = Coroutine_Yield(NULL, Coroutine_ChainYield, NULL);
1544
 err = (Coroutine_Err)(uintptr_t)Coroutine_GetValue(cor);
1545
 if (!err && result){
1546
 *result = res;
1547
 }
6 months ago
1548
 }
2 months ago
1549
 Coroutine_Delete(cor);
6 months ago
1550
 // success! ...probably
1551
 return err;
11 months ago
1552
 }
1553
1554
2 months ago
1555
 static Coroutine *
1556
 Coroutine_BiggestFreeBlock_Lock_Assumed(
Last month
1557
 void
2 months ago
1558
 ){
1559
 // check free list
1560
 Coroutine *best = NULL;
1561
 size_t best_size = 0;
1562
 List_Link *link;
1563
 for (link = List_Begin(&g_c->free); Link_NextIsLink(link); link = Link_Next(link)){
1564
 Coroutine *cor = List_Link_Container(Coroutine, link, link);
1565
 size_t cor_size = Coroutine_Size(cor);
1566
 if (cor_size > best_size){
1567
 best = cor;
1568
 best_size = cor_size;
1569
 }
1570
 }
1571
1572
 return best;
1573
 }
1574
1575
1576
 Coroutine_Err
1577
 Coroutine_CallWithMaxStack(
1578
 Coroutine_Start start,
1579
 void *value,
1580
 void **result
1581
 ){
1582
 MyAssert(!Coroutine_StackHasOverrun());
1583
 size_t headroom = (size_t)Coroutine_GetStackHeadroom();
1584
 if (headroom != PTRDIFF_MAX){
1585
 _Cor_Mutex_Lock(&g_c->mutex);
1586
1587
 Coroutine *cor = Coroutine_BiggestFreeBlock_Lock_Assumed();
1588
1589
 if (cor && Coroutine_Size(cor) > headroom){
1590
 // use cor for chaining
1591
 Coroutine *active = Coroutine_GetActive();
1592
 cor->min_size = active->min_size;
1593
 cor->min_headroom = active->min_headroom;
1594
 cor->state = Coroutine_Idle;
1595
 cor->start = Coroutine_ChainFn;
1596
 cor->value = NULL;
Last month
1597
 cor->chain_root = g_c->active->chain_root;
1598
 cor->chain_tip = cor;
1599
 g_c->active->chain_tip = cor;
2 months ago
1600
 Link_Remove(&cor->link);
1601
 List_AddHead(&g_c->inactive, &cor->link);
1602
 g_c->report.coroutines_created += 1;
1603
 } else {
1604
 cor = NULL;
1605
 }
1606
1607
 _Cor_Mutex_Unlock(&g_c->mutex);
1608
1609
 if (cor){
1610
 // we're chaining, not calling
1611
 struct Coroutine_ChainParam params = {
1612
 start,
1613
 value,
1614
 g_c->active
1615
 };
1616
 Coroutine_Err err = Coroutine_Continue(cor, &params, true);
1617
 if (!err){
1618
 void *res = Coroutine_Yield(NULL, Coroutine_ChainYield, NULL);
1619
 err = (Coroutine_Err)(uintptr_t)Coroutine_GetValue(cor);
1620
 if (!err && result){
1621
 *result = res;
1622
 }
1623
 }
1624
 Coroutine_Delete(cor);
1625
 return err;
1626
 }
1627
 }
1628
 void *ret = start(value);
1629
 if (result){
1630
 *result = ret;
1631
 }
1632
 return Coroutine_OK;
1633
 }
1634
1635
3 months ago
1636
 bool
2 months ago
1637
 Coroutine_IsRunning(
11 months ago
1638
 Coroutine *cor
3 months ago
1639
 ){
Last month
1640
 int state = cor->chain_tip->state;
3 months ago
1641
 return state == Coroutine_Running || state == Coroutine_Waiting;
11 months ago
4
1642
 }
11 months ago
1643
1644
2 months ago
1645
 bool Coroutine_IsComplete(
9 months ago
1646
 Coroutine *cor
3 months ago
1647
 ){
Last month
1648
 int state = cor->chain_tip->state;
3 months ago
1649
 return state == Coroutine_Complete;
9 months ago
1650
 }
1651
1652
3 months ago
1653
 bool
2 months ago
1654
 Coroutine_IsStarted(
3 months ago
1655
 void
1656
 ){
2 months ago
1657
 return g_c && g_c->state == Coroutines_Started;
11 months ago
1658
 }
8 months ago
1659
3 months ago
1660
 void
2 months ago
1661
 Coroutine_Dump_(
3 months ago
1662
 void
1663
 ){
8 months ago
1664
 char *state_to_text[] = {
1665
 "Free",
1666
 "Idle",
1667
 "Running",
1668
 "Waiting",
1669
 "Complete"
1670
 };
1671
 unsigned idx = 0;
1672
 List_Link *link;
1673
 for (link = List_Begin(&g_c->all); Link_NextIsLink(link); link = Link_Next(link)){
1674
 Coroutine *cor = List_Link_Container(Coroutine, all_link, link);
2 months ago
1675
 printf("%d) %p %p %zu (%s) %s\n", idx++, cor, cor->base, StackPointerDiff(cor->limit, cor->base), state_to_text[cor->state], cor == g_c->tip ? " (TIP)" : "");
8 months ago
1676
 }
1677
 }
2 months ago
1678
 #include "coroutine_names_undef.h"
1679