[{"id":"/root","type":"ROOT","loc":{"start":0,"end":5318,"column":{"s":0,"e":32},"line":{"s":0,"e":221}},"dim":[""],"code":null},{"id":"/root/children/0","type":"paragraph","loc":{"start":0,"end":98,"line":{"s":1,"e":1,"code":[";{ engine: dot, rankdir: TD, applyCss: dark, useGvPack: true, clusterize: false, intention: full }"]},"column":{"s":0,"e":98}},"dim":["","paragraph.0"],"code":";{ engine: dot, rankdir: TD, applyCss: dark, useGvPack: true, clusterize: false, intention: full }"},{"id":"/root/children/0/children/0","type":"text","loc":{"start":0,"end":98,"line":{"s":1,"e":1,"code":[";{ engine: dot, rankdir: TD, applyCss: dark, useGvPack: true, clusterize: false, intention: full }"]},"column":{"s":0,"e":98}},"dim":["","paragraph.0","text.0"],"code":";{ engine: dot, rankdir: TD, applyCss: dark, useGvPack: true, clusterize: false, intention: full }"},{"id":"/root/children/1","type":"heading","loc":{"start":100,"end":123,"line":{"s":3,"e":3,"code":["# pattern: orchestrator"]},"column":{"s":0,"e":23}},"dim":["","heading.1"],"code":"# pattern: orchestrator","symbName":"heading","symbRange":[125,273],"symbRangeL":[3,11],"outerCode":"\n- uses 3 queues\n- structure: serial, exclusive+parallel, serial\n- example:\n - `[ serial tasks ]`\n - `[ bag of parallel tasks ]`\n - `[ untiloop ]`","outerHtml":"\n"},{"id":"/root/children/1/children/0","type":"text","loc":{"start":102,"end":123,"line":{"s":3,"e":3,"code":["# pattern: orchestrator"]},"column":{"s":2,"e":23}},"dim":["","heading.1","text.0"],"code":"pattern: orchestrator"},{"id":"/root/children/2","type":"list","loc":{"start":125,"end":273,"line":{"s":5,"e":10,"code":["- uses 3 queues","- structure: serial, exclusive+parallel, serial","- example:"," - `[ serial tasks ]`"," - `[ bag of parallel tasks ]`"," - `[ untiloop ]`"]},"column":{"s":0,"e":18}},"dim":["","list.2"],"code":"- uses 3 queues\n- structure: serial, exclusive+parallel, serial\n- example:\n - `[ serial tasks ]`\n - `[ bag of parallel tasks ]`\n - `[ untiloop ]`","symbName":"list","symbRange":[275,290],"symbRangeL":[5,13],"outerCode":"- structure: serial, exclusive+parallel, serial\n- example:\n - `[ serial tasks ]`\n - `[ bag of parallel tasks ]`\n - `[ untiloop ]`\n\n## experimental","outerHtml":"\n\n

experimental

"},{"id":"/root/children/2/children/0","type":"listItem","loc":{"start":125,"end":140,"line":{"s":5,"e":5,"code":["- uses 3 queues"]},"column":{"s":0,"e":15}},"dim":["","list.2","listItem.0"],"code":"- uses 3 queues"},{"id":"/root/children/2/children/0/children/0","type":"paragraph","loc":{"start":127,"end":140,"line":{"s":5,"e":5,"code":["- uses 3 queues"]},"column":{"s":2,"e":15}},"dim":["","list.2","listItem.0","paragraph.0"],"code":"uses 3 queues"},{"id":"/root/children/2/children/0/children/0/children/0","type":"text","loc":{"start":127,"end":140,"line":{"s":5,"e":5,"code":["- uses 3 queues"]},"column":{"s":2,"e":15}},"dim":["","list.2","listItem.0","paragraph.0","text.0"],"code":"uses 3 queues"},{"id":"/root/children/2/children/1","type":"listItem","loc":{"start":141,"end":188,"line":{"s":6,"e":6,"code":["- structure: serial, exclusive+parallel, serial"]},"column":{"s":0,"e":47}},"dim":["","list.2","listItem.1"],"code":"- structure: serial, exclusive+parallel, serial"},{"id":"/root/children/2/children/1/children/0","type":"paragraph","loc":{"start":143,"end":188,"line":{"s":6,"e":6,"code":["- structure: serial, exclusive+parallel, serial"]},"column":{"s":2,"e":47}},"dim":["","list.2","listItem.1","paragraph.0"],"code":"structure: serial, exclusive+parallel, serial"},{"id":"/root/children/2/children/1/children/0/children/0","type":"text","loc":{"start":143,"end":188,"line":{"s":6,"e":6,"code":["- structure: serial, exclusive+parallel, serial"]},"column":{"s":2,"e":47}},"dim":["","list.2","listItem.1","paragraph.0","text.0"],"code":"structure: serial, exclusive+parallel, serial"},{"id":"/root/children/2/children/2","type":"listItem","loc":{"start":189,"end":273,"line":{"s":7,"e":10,"code":["- example:"," - `[ serial tasks ]`"," - `[ bag of parallel tasks ]`"," - `[ untiloop ]`"]},"column":{"s":0,"e":18}},"dim":["","list.2","listItem.2"],"code":"- example:\n - `[ serial tasks ]`\n - `[ bag of parallel tasks ]`\n - `[ untiloop ]`"},{"id":"/root/children/2/children/2/children/0","type":"paragraph","loc":{"start":191,"end":199,"line":{"s":7,"e":7,"code":["- example:"]},"column":{"s":2,"e":10}},"dim":["","list.2","listItem.2","paragraph.0"],"code":"example:"},{"id":"/root/children/2/children/2/children/0/children/0","type":"text","loc":{"start":191,"end":199,"line":{"s":7,"e":7,"code":["- example:"]},"column":{"s":2,"e":10}},"dim":["","list.2","listItem.2","paragraph.0","text.0"],"code":"example:"},{"id":"/root/children/2/children/2/children/1","type":"list","loc":{"start":202,"end":273,"line":{"s":8,"e":10,"code":[" - `[ serial tasks ]`"," - `[ bag of parallel tasks ]`"," - `[ untiloop ]`"]},"column":{"s":2,"e":18}},"dim":["","list.2","listItem.2","list.1"],"code":"- `[ serial tasks ]`\n - `[ bag of parallel tasks ]`\n - `[ untiloop ]`","symbName":"list","symbRange":[null,null],"symbRangeL":[8,10],"outerCode":" - `[ bag of parallel tasks ]`","outerHtml":""},{"id":"/root/children/2/children/2/children/1/children/0","type":"listItem","loc":{"start":202,"end":222,"line":{"s":8,"e":8,"code":[" - `[ serial tasks ]`"]},"column":{"s":2,"e":22}},"dim":["","list.2","listItem.2","list.1","listItem.0"],"code":"- `[ serial tasks ]`"},{"id":"/root/children/2/children/2/children/1/children/0/children/0","type":"paragraph","loc":{"start":204,"end":222,"line":{"s":8,"e":8,"code":[" - `[ serial tasks ]`"]},"column":{"s":4,"e":22}},"dim":["","list.2","listItem.2","list.1","listItem.0","paragraph.0"],"code":"`[ serial tasks ]`"},{"id":"/root/children/2/children/2/children/1/children/0/children/0/children/0","type":"inlineCode","loc":{"start":204,"end":222,"line":{"s":8,"e":8,"code":[" - `[ serial tasks ]`"]},"column":{"s":4,"e":22}},"dim":["","list.2","listItem.2","list.1","listItem.0","paragraph.0","inlineCode.0"],"code":"`[ serial tasks ]`"},{"id":"/root/children/2/children/2/children/1/children/1","type":"listItem","loc":{"start":225,"end":254,"line":{"s":9,"e":9,"code":[" - `[ bag of parallel tasks ]`"]},"column":{"s":2,"e":31}},"dim":["","list.2","listItem.2","list.1","listItem.1"],"code":"- `[ bag of parallel tasks ]`"},{"id":"/root/children/2/children/2/children/1/children/1/children/0","type":"paragraph","loc":{"start":227,"end":254,"line":{"s":9,"e":9,"code":[" - `[ bag of parallel tasks ]`"]},"column":{"s":4,"e":31}},"dim":["","list.2","listItem.2","list.1","listItem.1","paragraph.0"],"code":"`[ bag of parallel tasks ]`"},{"id":"/root/children/2/children/2/children/1/children/1/children/0/children/0","type":"inlineCode","loc":{"start":227,"end":254,"line":{"s":9,"e":9,"code":[" - `[ bag of parallel tasks ]`"]},"column":{"s":4,"e":31}},"dim":["","list.2","listItem.2","list.1","listItem.1","paragraph.0","inlineCode.0"],"code":"`[ bag of parallel tasks ]`"},{"id":"/root/children/2/children/2/children/1/children/2","type":"listItem","loc":{"start":257,"end":273,"line":{"s":10,"e":10,"code":[" - `[ untiloop ]`"]},"column":{"s":2,"e":18}},"dim":["","list.2","listItem.2","list.1","listItem.2"],"code":"- `[ untiloop ]`"},{"id":"/root/children/2/children/2/children/1/children/2/children/0","type":"paragraph","loc":{"start":259,"end":273,"line":{"s":10,"e":10,"code":[" - `[ untiloop ]`"]},"column":{"s":4,"e":18}},"dim":["","list.2","listItem.2","list.1","listItem.2","paragraph.0"],"code":"`[ untiloop ]`"},{"id":"/root/children/2/children/2/children/1/children/2/children/0/children/0","type":"inlineCode","loc":{"start":259,"end":273,"line":{"s":10,"e":10,"code":[" - `[ untiloop ]`"]},"column":{"s":4,"e":18}},"dim":["","list.2","listItem.2","list.1","listItem.2","paragraph.0","inlineCode.0"],"code":"`[ untiloop ]`"},{"id":"/root/children/3","type":"heading","loc":{"start":275,"end":290,"line":{"s":12,"e":12,"code":["## experimental"]},"column":{"s":0,"e":15}},"dim":["","heading.3"],"code":"## experimental","symbName":"heading","symbRange":[292,448],"symbRangeL":[12,17],"outerCode":"\n- TODO: could also support eval `code` ?\n- holds program state in namespaced redis keys: flat json\n- state can collect output of tasks in adjacent structure","outerHtml":"\n"},{"id":"/root/children/3/children/0","type":"text","loc":{"start":278,"end":290,"line":{"s":12,"e":12,"code":["## experimental"]},"column":{"s":3,"e":15}},"dim":["","heading.3","text.0"],"code":"experimental"},{"id":"/root/children/4","type":"list","loc":{"start":292,"end":448,"line":{"s":14,"e":16,"code":["- TODO: could also support eval `code` ?","- holds program state in namespaced redis keys: flat json","- state can collect output of tasks in adjacent structure"]},"column":{"s":0,"e":57}},"dim":["","list.4"],"code":"- TODO: could also support eval `code` ?\n- holds program state in namespaced redis keys: flat json\n- state can collect output of tasks in adjacent structure","symbName":"list","symbRange":[450,495],"symbRangeL":[14,19],"outerCode":"- holds program state in namespaced redis keys: flat json\n- state can collect output of tasks in adjacent structure\n\n## Recommended minimal clean setup (3 queues)","outerHtml":"\n\n

Recommended minimal clean setup (3 queues)

"},{"id":"/root/children/4/children/0","type":"listItem","loc":{"start":292,"end":332,"line":{"s":14,"e":14,"code":["- TODO: could also support eval `code` ?"]},"column":{"s":0,"e":40}},"dim":["","list.4","listItem.0"],"code":"- TODO: could also support eval `code` ?"},{"id":"/root/children/4/children/0/children/0","type":"paragraph","loc":{"start":294,"end":332,"line":{"s":14,"e":14,"code":["- TODO: could also support eval `code` ?"]},"column":{"s":2,"e":40}},"dim":["","list.4","listItem.0","paragraph.0"],"code":"TODO: could also support eval `code` ?"},{"id":"/root/children/4/children/0/children/0/children/0","type":"text","loc":{"start":294,"end":324,"line":{"s":14,"e":14,"code":["- TODO: could also support eval `code` ?"]},"column":{"s":2,"e":32}},"dim":["","list.4","listItem.0","paragraph.0","text.0"],"code":"TODO: could also support eval "},{"id":"/root/children/4/children/0/children/0/children/1","type":"inlineCode","loc":{"start":324,"end":330,"line":{"s":14,"e":14,"code":["- TODO: could also support eval `code` ?"]},"column":{"s":32,"e":38}},"dim":["","list.4","listItem.0","paragraph.0","inlineCode.1"],"code":"`code`"},{"id":"/root/children/4/children/0/children/0/children/2","type":"text","loc":{"start":330,"end":332,"line":{"s":14,"e":14,"code":["- TODO: could also support eval `code` ?"]},"column":{"s":38,"e":40}},"dim":["","list.4","listItem.0","paragraph.0","text.2"],"code":" ?"},{"id":"/root/children/4/children/1","type":"listItem","loc":{"start":333,"end":390,"line":{"s":15,"e":15,"code":["- holds program state in namespaced redis keys: flat json"]},"column":{"s":0,"e":57}},"dim":["","list.4","listItem.1"],"code":"- holds program state in namespaced redis keys: flat json"},{"id":"/root/children/4/children/1/children/0","type":"paragraph","loc":{"start":335,"end":390,"line":{"s":15,"e":15,"code":["- holds program state in namespaced redis keys: flat json"]},"column":{"s":2,"e":57}},"dim":["","list.4","listItem.1","paragraph.0"],"code":"holds program state in namespaced redis keys: flat json"},{"id":"/root/children/4/children/1/children/0/children/0","type":"text","loc":{"start":335,"end":390,"line":{"s":15,"e":15,"code":["- holds program state in namespaced redis keys: flat json"]},"column":{"s":2,"e":57}},"dim":["","list.4","listItem.1","paragraph.0","text.0"],"code":"holds program state in namespaced redis keys: flat json"},{"id":"/root/children/4/children/2","type":"listItem","loc":{"start":391,"end":448,"line":{"s":16,"e":16,"code":["- state can collect output of tasks in adjacent structure"]},"column":{"s":0,"e":57}},"dim":["","list.4","listItem.2"],"code":"- state can collect output of tasks in adjacent structure"},{"id":"/root/children/4/children/2/children/0","type":"paragraph","loc":{"start":393,"end":448,"line":{"s":16,"e":16,"code":["- state can collect output of tasks in adjacent structure"]},"column":{"s":2,"e":57}},"dim":["","list.4","listItem.2","paragraph.0"],"code":"state can collect output of tasks in adjacent structure"},{"id":"/root/children/4/children/2/children/0/children/0","type":"text","loc":{"start":393,"end":448,"line":{"s":16,"e":16,"code":["- state can collect output of tasks in adjacent structure"]},"column":{"s":2,"e":57}},"dim":["","list.4","listItem.2","paragraph.0","text.0"],"code":"state can collect output of tasks in adjacent structure"},{"id":"/root/children/5","type":"heading","loc":{"start":450,"end":495,"line":{"s":18,"e":18,"code":["## Recommended minimal clean setup (3 queues)"]},"column":{"s":0,"e":45}},"dim":["","heading.5"],"code":"## Recommended minimal clean setup (3 queues)","symbName":"heading","symbRange":[497,1108],"symbRangeL":[18,43],"outerCode":"\n- /queue/orchestrator\n - 100% serial queue: prefetch = 1, 1 consumer only\n - State machine messages only (continue workflow, step done, parallel done,\n start workflow, etc.)\n\n- /queue/serial-tasks\n - 100% serial queue: prefetch = 1, 1 consumer only\n - tasks run sequentially by nature\n\n- /queue/parallel-tasks\n - many consumers\n - tasks run concurrently\n\n- each task updates its state after finished\n - TODO: but how?\n - completion message\n\nThis is the simplest clean design that gives you:\n\n- controlled sequential execution\n- unlimited parallelism\n- simple, readable routing\n- non-interfering logic","outerHtml":"\n\n\n\n\n\n\n\n\n

This is the simplest clean design that gives you:

\n\n"},{"id":"/root/children/5/children/0","type":"text","loc":{"start":453,"end":495,"line":{"s":18,"e":18,"code":["## Recommended minimal clean setup (3 queues)"]},"column":{"s":3,"e":45}},"dim":["","heading.5","text.0"],"code":"Recommended minimal clean setup (3 queues)"},{"id":"/root/children/6","type":"list","loc":{"start":497,"end":947,"line":{"s":20,"e":35,"code":["- /queue/orchestrator"," - 100% serial queue: prefetch = 1, 1 consumer only"," - State machine messages only (continue workflow, step done, parallel done,"," start workflow, etc.)","","- /queue/serial-tasks"," - 100% serial queue: prefetch = 1, 1 consumer only"," - tasks run sequentially by nature","","- /queue/parallel-tasks"," - many consumers"," - tasks run concurrently","","- each task updates its state after finished"," - TODO: but how?"," - completion message"]},"column":{"s":0,"e":22}},"dim":["","list.6"],"code":"- /queue/orchestrator\n - 100% serial queue: prefetch = 1, 1 consumer only\n - State machine messages only (continue workflow, step done, parallel done,\n start workflow, etc.)\n\n- /queue/serial-tasks\n - 100% serial queue: prefetch = 1, 1 consumer only\n - tasks run sequentially by nature\n\n- /queue/parallel-tasks\n - many consumers\n - tasks run concurrently\n\n- each task updates its state after finished\n - TODO: but how?\n - completion message","symbName":"list","symbRange":[949,998],"symbRangeL":[20,38],"outerCode":" - 100% serial queue: prefetch = 1, 1 consumer only\n - State machine messages only (continue workflow, step done, parallel done,\n start workflow, etc.)\n\n- /queue/serial-tasks\n - 100% serial queue: prefetch = 1, 1 consumer only\n - tasks run sequentially by nature\n\n- /queue/parallel-tasks\n - many consumers\n - tasks run concurrently\n\n- each task updates its state after finished\n - TODO: but how?\n - completion message\n\nThis is the simplest clean design that gives you:","outerHtml":"\n\n\n\n\n\n\n\n

This is the simplest clean design that gives you:

"},{"id":"/root/children/6/children/0","type":"listItem","loc":{"start":497,"end":675,"line":{"s":20,"e":23,"code":["- /queue/orchestrator"," - 100% serial queue: prefetch = 1, 1 consumer only"," - State machine messages only (continue workflow, step done, parallel done,"," start workflow, etc.)"]},"column":{"s":0,"e":25}},"dim":["","list.6","listItem.0"],"code":"- /queue/orchestrator\n - 100% serial queue: prefetch = 1, 1 consumer only\n - State machine messages only (continue workflow, step done, parallel done,\n start workflow, etc.)"},{"id":"/root/children/6/children/0/children/0","type":"paragraph","loc":{"start":499,"end":518,"line":{"s":20,"e":20,"code":["- /queue/orchestrator"]},"column":{"s":2,"e":21}},"dim":["","list.6","listItem.0","paragraph.0"],"code":"/queue/orchestrator"},{"id":"/root/children/6/children/0/children/0/children/0","type":"text","loc":{"start":499,"end":518,"line":{"s":20,"e":20,"code":["- /queue/orchestrator"]},"column":{"s":2,"e":21}},"dim":["","list.6","listItem.0","paragraph.0","text.0"],"code":"/queue/orchestrator"},{"id":"/root/children/6/children/0/children/1","type":"list","loc":{"start":521,"end":675,"line":{"s":21,"e":23,"code":[" - 100% serial queue: prefetch = 1, 1 consumer only"," - State machine messages only (continue workflow, step done, parallel done,"," start workflow, etc.)"]},"column":{"s":2,"e":25}},"dim":["","list.6","listItem.0","list.1"],"code":"- 100% serial queue: prefetch = 1, 1 consumer only\n - State machine messages only (continue workflow, step done, parallel done,\n start workflow, etc.)","symbName":"list","symbRange":[null,null],"symbRangeL":[21,23],"outerCode":" - State machine messages only (continue workflow, step done, parallel done,","outerHtml":""},{"id":"/root/children/6/children/0/children/1/children/0","type":"listItem","loc":{"start":521,"end":571,"line":{"s":21,"e":21,"code":[" - 100% serial queue: prefetch = 1, 1 consumer only"]},"column":{"s":2,"e":52}},"dim":["","list.6","listItem.0","list.1","listItem.0"],"code":"- 100% serial queue: prefetch = 1, 1 consumer only"},{"id":"/root/children/6/children/0/children/1/children/0/children/0","type":"paragraph","loc":{"start":523,"end":571,"line":{"s":21,"e":21,"code":[" - 100% serial queue: prefetch = 1, 1 consumer only"]},"column":{"s":4,"e":52}},"dim":["","list.6","listItem.0","list.1","listItem.0","paragraph.0"],"code":"100% serial queue: prefetch = 1, 1 consumer only"},{"id":"/root/children/6/children/0/children/1/children/0/children/0/children/0","type":"text","loc":{"start":523,"end":571,"line":{"s":21,"e":21,"code":[" - 100% serial queue: prefetch = 1, 1 consumer only"]},"column":{"s":4,"e":52}},"dim":["","list.6","listItem.0","list.1","listItem.0","paragraph.0","text.0"],"code":"100% serial queue: prefetch = 1, 1 consumer only"},{"id":"/root/children/6/children/0/children/1/children/1","type":"listItem","loc":{"start":574,"end":675,"line":{"s":22,"e":23,"code":[" - State machine messages only (continue workflow, step done, parallel done,"," start workflow, etc.)"]},"column":{"s":2,"e":25}},"dim":["","list.6","listItem.0","list.1","listItem.1"],"code":"- State machine messages only (continue workflow, step done, parallel done,\n start workflow, etc.)"},{"id":"/root/children/6/children/0/children/1/children/1/children/0","type":"paragraph","loc":{"start":576,"end":675,"line":{"s":22,"e":23,"code":[" - State machine messages only (continue workflow, step done, parallel done,"," start workflow, etc.)"]},"column":{"s":4,"e":25}},"dim":["","list.6","listItem.0","list.1","listItem.1","paragraph.0"],"code":"State machine messages only (continue workflow, step done, parallel done,\n start workflow, etc.)"},{"id":"/root/children/6/children/0/children/1/children/1/children/0/children/0","type":"text","loc":{"start":576,"end":675,"line":{"s":22,"e":23,"code":[" - State machine messages only (continue workflow, step done, parallel done,"," start workflow, etc.)"]},"column":{"s":4,"e":25}},"dim":["","list.6","listItem.0","list.1","listItem.1","paragraph.0","text.0"],"code":"State machine messages only (continue workflow, step done, parallel done,\n start workflow, etc.)"},{"id":"/root/children/6/children/1","type":"listItem","loc":{"start":677,"end":788,"line":{"s":25,"e":27,"code":["- /queue/serial-tasks"," - 100% serial queue: prefetch = 1, 1 consumer only"," - tasks run sequentially by nature"]},"column":{"s":0,"e":36}},"dim":["","list.6","listItem.1"],"code":"- /queue/serial-tasks\n - 100% serial queue: prefetch = 1, 1 consumer only\n - tasks run sequentially by nature"},{"id":"/root/children/6/children/1/children/0","type":"paragraph","loc":{"start":679,"end":698,"line":{"s":25,"e":25,"code":["- /queue/serial-tasks"]},"column":{"s":2,"e":21}},"dim":["","list.6","listItem.1","paragraph.0"],"code":"/queue/serial-tasks"},{"id":"/root/children/6/children/1/children/0/children/0","type":"text","loc":{"start":679,"end":698,"line":{"s":25,"e":25,"code":["- /queue/serial-tasks"]},"column":{"s":2,"e":21}},"dim":["","list.6","listItem.1","paragraph.0","text.0"],"code":"/queue/serial-tasks"},{"id":"/root/children/6/children/1/children/1","type":"list","loc":{"start":701,"end":788,"line":{"s":26,"e":27,"code":[" - 100% serial queue: prefetch = 1, 1 consumer only"," - tasks run sequentially by nature"]},"column":{"s":2,"e":36}},"dim":["","list.6","listItem.1","list.1"],"code":"- 100% serial queue: prefetch = 1, 1 consumer only\n - tasks run sequentially by nature","symbName":"list","symbRange":[null,null],"symbRangeL":[26,27],"outerCode":"","outerHtml":""},{"id":"/root/children/6/children/1/children/1/children/0","type":"listItem","loc":{"start":701,"end":751,"line":{"s":26,"e":26,"code":[" - 100% serial queue: prefetch = 1, 1 consumer only"]},"column":{"s":2,"e":52}},"dim":["","list.6","listItem.1","list.1","listItem.0"],"code":"- 100% serial queue: prefetch = 1, 1 consumer only"},{"id":"/root/children/6/children/1/children/1/children/0/children/0","type":"paragraph","loc":{"start":703,"end":751,"line":{"s":26,"e":26,"code":[" - 100% serial queue: prefetch = 1, 1 consumer only"]},"column":{"s":4,"e":52}},"dim":["","list.6","listItem.1","list.1","listItem.0","paragraph.0"],"code":"100% serial queue: prefetch = 1, 1 consumer only"},{"id":"/root/children/6/children/1/children/1/children/0/children/0/children/0","type":"text","loc":{"start":703,"end":751,"line":{"s":26,"e":26,"code":[" - 100% serial queue: prefetch = 1, 1 consumer only"]},"column":{"s":4,"e":52}},"dim":["","list.6","listItem.1","list.1","listItem.0","paragraph.0","text.0"],"code":"100% serial queue: prefetch = 1, 1 consumer only"},{"id":"/root/children/6/children/1/children/1/children/1","type":"listItem","loc":{"start":754,"end":788,"line":{"s":27,"e":27,"code":[" - tasks run sequentially by nature"]},"column":{"s":2,"e":36}},"dim":["","list.6","listItem.1","list.1","listItem.1"],"code":"- tasks run sequentially by nature"},{"id":"/root/children/6/children/1/children/1/children/1/children/0","type":"paragraph","loc":{"start":756,"end":788,"line":{"s":27,"e":27,"code":[" - tasks run sequentially by nature"]},"column":{"s":4,"e":36}},"dim":["","list.6","listItem.1","list.1","listItem.1","paragraph.0"],"code":"tasks run sequentially by nature"},{"id":"/root/children/6/children/1/children/1/children/1/children/0/children/0","type":"text","loc":{"start":756,"end":788,"line":{"s":27,"e":27,"code":[" - tasks run sequentially by nature"]},"column":{"s":4,"e":36}},"dim":["","list.6","listItem.1","list.1","listItem.1","paragraph.0","text.0"],"code":"tasks run sequentially by nature"},{"id":"/root/children/6/children/2","type":"listItem","loc":{"start":790,"end":859,"line":{"s":29,"e":31,"code":["- /queue/parallel-tasks"," - many consumers"," - tasks run concurrently"]},"column":{"s":0,"e":26}},"dim":["","list.6","listItem.2"],"code":"- /queue/parallel-tasks\n - many consumers\n - tasks run concurrently"},{"id":"/root/children/6/children/2/children/0","type":"paragraph","loc":{"start":792,"end":813,"line":{"s":29,"e":29,"code":["- /queue/parallel-tasks"]},"column":{"s":2,"e":23}},"dim":["","list.6","listItem.2","paragraph.0"],"code":"/queue/parallel-tasks"},{"id":"/root/children/6/children/2/children/0/children/0","type":"text","loc":{"start":792,"end":813,"line":{"s":29,"e":29,"code":["- /queue/parallel-tasks"]},"column":{"s":2,"e":23}},"dim":["","list.6","listItem.2","paragraph.0","text.0"],"code":"/queue/parallel-tasks"},{"id":"/root/children/6/children/2/children/1","type":"list","loc":{"start":816,"end":859,"line":{"s":30,"e":31,"code":[" - many consumers"," - tasks run concurrently"]},"column":{"s":2,"e":26}},"dim":["","list.6","listItem.2","list.1"],"code":"- many consumers\n - tasks run concurrently","symbName":"list","symbRange":[null,null],"symbRangeL":[30,31],"outerCode":"","outerHtml":""},{"id":"/root/children/6/children/2/children/1/children/0","type":"listItem","loc":{"start":816,"end":832,"line":{"s":30,"e":30,"code":[" - many consumers"]},"column":{"s":2,"e":18}},"dim":["","list.6","listItem.2","list.1","listItem.0"],"code":"- many consumers"},{"id":"/root/children/6/children/2/children/1/children/0/children/0","type":"paragraph","loc":{"start":818,"end":832,"line":{"s":30,"e":30,"code":[" - many consumers"]},"column":{"s":4,"e":18}},"dim":["","list.6","listItem.2","list.1","listItem.0","paragraph.0"],"code":"many consumers"},{"id":"/root/children/6/children/2/children/1/children/0/children/0/children/0","type":"text","loc":{"start":818,"end":832,"line":{"s":30,"e":30,"code":[" - many consumers"]},"column":{"s":4,"e":18}},"dim":["","list.6","listItem.2","list.1","listItem.0","paragraph.0","text.0"],"code":"many consumers"},{"id":"/root/children/6/children/2/children/1/children/1","type":"listItem","loc":{"start":835,"end":859,"line":{"s":31,"e":31,"code":[" - tasks run concurrently"]},"column":{"s":2,"e":26}},"dim":["","list.6","listItem.2","list.1","listItem.1"],"code":"- tasks run concurrently"},{"id":"/root/children/6/children/2/children/1/children/1/children/0","type":"paragraph","loc":{"start":837,"end":859,"line":{"s":31,"e":31,"code":[" - tasks run concurrently"]},"column":{"s":4,"e":26}},"dim":["","list.6","listItem.2","list.1","listItem.1","paragraph.0"],"code":"tasks run concurrently"},{"id":"/root/children/6/children/2/children/1/children/1/children/0/children/0","type":"text","loc":{"start":837,"end":859,"line":{"s":31,"e":31,"code":[" - tasks run concurrently"]},"column":{"s":4,"e":26}},"dim":["","list.6","listItem.2","list.1","listItem.1","paragraph.0","text.0"],"code":"tasks run concurrently"},{"id":"/root/children/6/children/3","type":"listItem","loc":{"start":861,"end":947,"line":{"s":33,"e":35,"code":["- each task updates its state after finished"," - TODO: but how?"," - completion message"]},"column":{"s":0,"e":22}},"dim":["","list.6","listItem.3"],"code":"- each task updates its state after finished\n - TODO: but how?\n - completion message"},{"id":"/root/children/6/children/3/children/0","type":"paragraph","loc":{"start":863,"end":905,"line":{"s":33,"e":33,"code":["- each task updates its state after finished"]},"column":{"s":2,"e":44}},"dim":["","list.6","listItem.3","paragraph.0"],"code":"each task updates its state after finished"},{"id":"/root/children/6/children/3/children/0/children/0","type":"text","loc":{"start":863,"end":905,"line":{"s":33,"e":33,"code":["- each task updates its state after finished"]},"column":{"s":2,"e":44}},"dim":["","list.6","listItem.3","paragraph.0","text.0"],"code":"each task updates its state after finished"},{"id":"/root/children/6/children/3/children/1","type":"list","loc":{"start":908,"end":947,"line":{"s":34,"e":35,"code":[" - TODO: but how?"," - completion message"]},"column":{"s":2,"e":22}},"dim":["","list.6","listItem.3","list.1"],"code":"- TODO: but how?\n - completion message","symbName":"list","symbRange":[null,null],"symbRangeL":[34,35],"outerCode":"","outerHtml":""},{"id":"/root/children/6/children/3/children/1/children/0","type":"listItem","loc":{"start":908,"end":924,"line":{"s":34,"e":34,"code":[" - TODO: but how?"]},"column":{"s":2,"e":18}},"dim":["","list.6","listItem.3","list.1","listItem.0"],"code":"- TODO: but how?"},{"id":"/root/children/6/children/3/children/1/children/0/children/0","type":"paragraph","loc":{"start":910,"end":924,"line":{"s":34,"e":34,"code":[" - TODO: but how?"]},"column":{"s":4,"e":18}},"dim":["","list.6","listItem.3","list.1","listItem.0","paragraph.0"],"code":"TODO: but how?"},{"id":"/root/children/6/children/3/children/1/children/0/children/0/children/0","type":"text","loc":{"start":910,"end":924,"line":{"s":34,"e":34,"code":[" - TODO: but how?"]},"column":{"s":4,"e":18}},"dim":["","list.6","listItem.3","list.1","listItem.0","paragraph.0","text.0"],"code":"TODO: but how?"},{"id":"/root/children/6/children/3/children/1/children/1","type":"listItem","loc":{"start":927,"end":947,"line":{"s":35,"e":35,"code":[" - completion message"]},"column":{"s":2,"e":22}},"dim":["","list.6","listItem.3","list.1","listItem.1"],"code":"- completion message"},{"id":"/root/children/6/children/3/children/1/children/1/children/0","type":"paragraph","loc":{"start":929,"end":947,"line":{"s":35,"e":35,"code":[" - completion message"]},"column":{"s":4,"e":22}},"dim":["","list.6","listItem.3","list.1","listItem.1","paragraph.0"],"code":"completion message"},{"id":"/root/children/6/children/3/children/1/children/1/children/0/children/0","type":"text","loc":{"start":929,"end":947,"line":{"s":35,"e":35,"code":[" - completion message"]},"column":{"s":4,"e":22}},"dim":["","list.6","listItem.3","list.1","listItem.1","paragraph.0","text.0"],"code":"completion message"},{"id":"/root/children/7","type":"paragraph","loc":{"start":949,"end":998,"line":{"s":37,"e":37,"code":["This is the simplest clean design that gives you:"]},"column":{"s":0,"e":49}},"dim":["","paragraph.7"],"code":"This is the simplest clean design that gives you:"},{"id":"/root/children/7/children/0","type":"text","loc":{"start":949,"end":998,"line":{"s":37,"e":37,"code":["This is the simplest clean design that gives you:"]},"column":{"s":0,"e":49}},"dim":["","paragraph.7","text.0"],"code":"This is the simplest clean design that gives you:"},{"id":"/root/children/8","type":"list","loc":{"start":1000,"end":1108,"line":{"s":39,"e":42,"code":["- controlled sequential execution","- unlimited parallelism","- simple, readable routing","- non-interfering logic"]},"column":{"s":0,"e":23}},"dim":["","list.8"],"code":"- controlled sequential execution\n- unlimited parallelism\n- simple, readable routing\n- non-interfering logic","symbName":"list","symbRange":[1110,1120],"symbRangeL":[39,45],"outerCode":"- unlimited parallelism\n- simple, readable routing\n- non-interfering logic\n\n### queues","outerHtml":"\n\n

queues

"},{"id":"/root/children/8/children/0","type":"listItem","loc":{"start":1000,"end":1033,"line":{"s":39,"e":39,"code":["- controlled sequential execution"]},"column":{"s":0,"e":33}},"dim":["","list.8","listItem.0"],"code":"- controlled sequential execution"},{"id":"/root/children/8/children/0/children/0","type":"paragraph","loc":{"start":1002,"end":1033,"line":{"s":39,"e":39,"code":["- controlled sequential execution"]},"column":{"s":2,"e":33}},"dim":["","list.8","listItem.0","paragraph.0"],"code":"controlled sequential execution"},{"id":"/root/children/8/children/0/children/0/children/0","type":"text","loc":{"start":1002,"end":1033,"line":{"s":39,"e":39,"code":["- controlled sequential execution"]},"column":{"s":2,"e":33}},"dim":["","list.8","listItem.0","paragraph.0","text.0"],"code":"controlled sequential execution"},{"id":"/root/children/8/children/1","type":"listItem","loc":{"start":1034,"end":1057,"line":{"s":40,"e":40,"code":["- unlimited parallelism"]},"column":{"s":0,"e":23}},"dim":["","list.8","listItem.1"],"code":"- unlimited parallelism"},{"id":"/root/children/8/children/1/children/0","type":"paragraph","loc":{"start":1036,"end":1057,"line":{"s":40,"e":40,"code":["- unlimited parallelism"]},"column":{"s":2,"e":23}},"dim":["","list.8","listItem.1","paragraph.0"],"code":"unlimited parallelism"},{"id":"/root/children/8/children/1/children/0/children/0","type":"text","loc":{"start":1036,"end":1057,"line":{"s":40,"e":40,"code":["- unlimited parallelism"]},"column":{"s":2,"e":23}},"dim":["","list.8","listItem.1","paragraph.0","text.0"],"code":"unlimited parallelism"},{"id":"/root/children/8/children/2","type":"listItem","loc":{"start":1058,"end":1084,"line":{"s":41,"e":41,"code":["- simple, readable routing"]},"column":{"s":0,"e":26}},"dim":["","list.8","listItem.2"],"code":"- simple, readable routing"},{"id":"/root/children/8/children/2/children/0","type":"paragraph","loc":{"start":1060,"end":1084,"line":{"s":41,"e":41,"code":["- simple, readable routing"]},"column":{"s":2,"e":26}},"dim":["","list.8","listItem.2","paragraph.0"],"code":"simple, readable routing"},{"id":"/root/children/8/children/2/children/0/children/0","type":"text","loc":{"start":1060,"end":1084,"line":{"s":41,"e":41,"code":["- simple, readable routing"]},"column":{"s":2,"e":26}},"dim":["","list.8","listItem.2","paragraph.0","text.0"],"code":"simple, readable routing"},{"id":"/root/children/8/children/3","type":"listItem","loc":{"start":1085,"end":1108,"line":{"s":42,"e":42,"code":["- non-interfering logic"]},"column":{"s":0,"e":23}},"dim":["","list.8","listItem.3"],"code":"- non-interfering logic"},{"id":"/root/children/8/children/3/children/0","type":"paragraph","loc":{"start":1087,"end":1108,"line":{"s":42,"e":42,"code":["- non-interfering logic"]},"column":{"s":2,"e":23}},"dim":["","list.8","listItem.3","paragraph.0"],"code":"non-interfering logic"},{"id":"/root/children/8/children/3/children/0/children/0","type":"text","loc":{"start":1087,"end":1108,"line":{"s":42,"e":42,"code":["- non-interfering logic"]},"column":{"s":2,"e":23}},"dim":["","list.8","listItem.3","paragraph.0","text.0"],"code":"non-interfering logic"},{"id":"/root/children/9","type":"heading","loc":{"start":1110,"end":1120,"line":{"s":44,"e":44,"code":["### queues"]},"column":{"s":0,"e":10}},"dim":["","heading.9"],"code":"### queues","symbName":"heading","symbRange":[1122,1186],"symbRangeL":[44,49],"outerCode":"\n- queue_orchestrator\n- queue_tasks_serial\n- queue_tasks_parallel","outerHtml":"\n"},{"id":"/root/children/9/children/0","type":"text","loc":{"start":1114,"end":1120,"line":{"s":44,"e":44,"code":["### queues"]},"column":{"s":4,"e":10}},"dim":["","heading.9","text.0"],"code":"queues"},{"id":"/root/children/10","type":"list","loc":{"start":1122,"end":1186,"line":{"s":46,"e":48,"code":["- queue_orchestrator","- queue_tasks_serial","- queue_tasks_parallel"]},"column":{"s":0,"e":22}},"dim":["","list.10"],"code":"- queue_orchestrator\n- queue_tasks_serial\n- queue_tasks_parallel","symbName":"list","symbRange":[1188,1429],"symbRangeL":[46,58],"outerCode":"- queue_tasks_serial\n- queue_tasks_parallel\n\n## Workflow Orchestration\n\nRabbitMQ is only the transport; the orchestration logic lives in\nthe program-json.\n\nThis is a “Workflow Orchestration” (a.k.a.\nSaga Orchestration) pattern, a classic Saga orchestration where every “step”\nis either:","outerHtml":"\n\n

Workflow Orchestration

\n\n

RabbitMQ is only the transport; the orchestration logic lives in\nthe program-json.

\n\n

This is a “Workflow Orchestration” (a.k.a.\nSaga Orchestration) pattern, a classic Saga orchestration where every “step”\nis either:

"},{"id":"/root/children/10/children/0","type":"listItem","loc":{"start":1122,"end":1142,"line":{"s":46,"e":46,"code":["- queue_orchestrator"]},"column":{"s":0,"e":20}},"dim":["","list.10","listItem.0"],"code":"- queue_orchestrator"},{"id":"/root/children/10/children/0/children/0","type":"paragraph","loc":{"start":1124,"end":1142,"line":{"s":46,"e":46,"code":["- queue_orchestrator"]},"column":{"s":2,"e":20}},"dim":["","list.10","listItem.0","paragraph.0"],"code":"queue_orchestrator"},{"id":"/root/children/10/children/0/children/0/children/0","type":"text","loc":{"start":1124,"end":1142,"line":{"s":46,"e":46,"code":["- queue_orchestrator"]},"column":{"s":2,"e":20}},"dim":["","list.10","listItem.0","paragraph.0","text.0"],"code":"queue_orchestrator"},{"id":"/root/children/10/children/1","type":"listItem","loc":{"start":1143,"end":1163,"line":{"s":47,"e":47,"code":["- queue_tasks_serial"]},"column":{"s":0,"e":20}},"dim":["","list.10","listItem.1"],"code":"- queue_tasks_serial"},{"id":"/root/children/10/children/1/children/0","type":"paragraph","loc":{"start":1145,"end":1163,"line":{"s":47,"e":47,"code":["- queue_tasks_serial"]},"column":{"s":2,"e":20}},"dim":["","list.10","listItem.1","paragraph.0"],"code":"queue_tasks_serial"},{"id":"/root/children/10/children/1/children/0/children/0","type":"text","loc":{"start":1145,"end":1163,"line":{"s":47,"e":47,"code":["- queue_tasks_serial"]},"column":{"s":2,"e":20}},"dim":["","list.10","listItem.1","paragraph.0","text.0"],"code":"queue_tasks_serial"},{"id":"/root/children/10/children/2","type":"listItem","loc":{"start":1164,"end":1186,"line":{"s":48,"e":48,"code":["- queue_tasks_parallel"]},"column":{"s":0,"e":22}},"dim":["","list.10","listItem.2"],"code":"- queue_tasks_parallel"},{"id":"/root/children/10/children/2/children/0","type":"paragraph","loc":{"start":1166,"end":1186,"line":{"s":48,"e":48,"code":["- queue_tasks_parallel"]},"column":{"s":2,"e":22}},"dim":["","list.10","listItem.2","paragraph.0"],"code":"queue_tasks_parallel"},{"id":"/root/children/10/children/2/children/0/children/0","type":"text","loc":{"start":1166,"end":1186,"line":{"s":48,"e":48,"code":["- queue_tasks_parallel"]},"column":{"s":2,"e":22}},"dim":["","list.10","listItem.2","paragraph.0","text.0"],"code":"queue_tasks_parallel"},{"id":"/root/children/11","type":"heading","loc":{"start":1188,"end":1213,"line":{"s":50,"e":50,"code":["## Workflow Orchestration"]},"column":{"s":0,"e":25}},"dim":["","heading.11"],"code":"## Workflow Orchestration","symbName":"heading","symbRange":[1215,1827],"symbRangeL":[50,69],"outerCode":"\nRabbitMQ is only the transport; the orchestration logic lives in\nthe program-json.\n\nThis is a “Workflow Orchestration” (a.k.a.\nSaga Orchestration) pattern, a classic Saga orchestration where every “step”\nis either:\n\n- a sequential action, or\n- a parallel fan-out group with a join barrier.\n\nOrchestrated Workflow with Barrier Synchronization\n\n- Reads the workflow definition (your array).\n- For each step:\n- If it’s a single task → publish to a serial queue, wait for ack.\n- If it’s a parallel block → fan-out messages to a parallel queue,\n track N acks, wait until all are done (barrier),continue to next step.","outerHtml":"\n

RabbitMQ is only the transport; the orchestration logic lives in\nthe program-json.

\n\n

This is a “Workflow Orchestration” (a.k.a.\nSaga Orchestration) pattern, a classic Saga orchestration where every “step”\nis either:

\n\n\n\n

Orchestrated Workflow with Barrier Synchronization

\n\n"},{"id":"/root/children/11/children/0","type":"text","loc":{"start":1191,"end":1213,"line":{"s":50,"e":50,"code":["## Workflow Orchestration"]},"column":{"s":3,"e":25}},"dim":["","heading.11","text.0"],"code":"Workflow Orchestration"},{"id":"/root/children/12","type":"paragraph","loc":{"start":1215,"end":1297,"line":{"s":52,"e":53,"code":["RabbitMQ is only the transport; the orchestration logic lives in","the program-json."]},"column":{"s":0,"e":17}},"dim":["","paragraph.12"],"code":"RabbitMQ is only the transport; the orchestration logic lives in\nthe program-json."},{"id":"/root/children/12/children/0","type":"text","loc":{"start":1215,"end":1297,"line":{"s":52,"e":53,"code":["RabbitMQ is only the transport; the orchestration logic lives in","the program-json."]},"column":{"s":0,"e":17}},"dim":["","paragraph.12","text.0"],"code":"RabbitMQ is only the transport; the orchestration logic lives in\nthe program-json."},{"id":"/root/children/13","type":"paragraph","loc":{"start":1299,"end":1429,"line":{"s":55,"e":57,"code":["This is a “Workflow Orchestration” (a.k.a.","Saga Orchestration) pattern, a classic Saga orchestration where every “step”","is either:"]},"column":{"s":0,"e":10}},"dim":["","paragraph.13"],"code":"This is a “Workflow Orchestration” (a.k.a.\nSaga Orchestration) pattern, a classic Saga orchestration where every “step”\nis either:"},{"id":"/root/children/13/children/0","type":"text","loc":{"start":1299,"end":1429,"line":{"s":55,"e":57,"code":["This is a “Workflow Orchestration” (a.k.a.","Saga Orchestration) pattern, a classic Saga orchestration where every “step”","is either:"]},"column":{"s":0,"e":10}},"dim":["","paragraph.13","text.0"],"code":"This is a “Workflow Orchestration” (a.k.a.\nSaga Orchestration) pattern, a classic Saga orchestration where every “step”\nis either:"},{"id":"/root/children/14","type":"list","loc":{"start":1431,"end":1504,"line":{"s":59,"e":60,"code":["- a sequential action, or","- a parallel fan-out group with a join barrier."]},"column":{"s":0,"e":47}},"dim":["","list.14"],"code":"- a sequential action, or\n- a parallel fan-out group with a join barrier.","symbName":"list","symbRange":[1506,1556],"symbRangeL":[59,63],"outerCode":"- a parallel fan-out group with a join barrier.\n\nOrchestrated Workflow with Barrier Synchronization","outerHtml":"\n\n

Orchestrated Workflow with Barrier Synchronization

"},{"id":"/root/children/14/children/0","type":"listItem","loc":{"start":1431,"end":1456,"line":{"s":59,"e":59,"code":["- a sequential action, or"]},"column":{"s":0,"e":25}},"dim":["","list.14","listItem.0"],"code":"- a sequential action, or"},{"id":"/root/children/14/children/0/children/0","type":"paragraph","loc":{"start":1433,"end":1456,"line":{"s":59,"e":59,"code":["- a sequential action, or"]},"column":{"s":2,"e":25}},"dim":["","list.14","listItem.0","paragraph.0"],"code":"a sequential action, or"},{"id":"/root/children/14/children/0/children/0/children/0","type":"text","loc":{"start":1433,"end":1456,"line":{"s":59,"e":59,"code":["- a sequential action, or"]},"column":{"s":2,"e":25}},"dim":["","list.14","listItem.0","paragraph.0","text.0"],"code":"a sequential action, or"},{"id":"/root/children/14/children/1","type":"listItem","loc":{"start":1457,"end":1504,"line":{"s":60,"e":60,"code":["- a parallel fan-out group with a join barrier."]},"column":{"s":0,"e":47}},"dim":["","list.14","listItem.1"],"code":"- a parallel fan-out group with a join barrier."},{"id":"/root/children/14/children/1/children/0","type":"paragraph","loc":{"start":1459,"end":1504,"line":{"s":60,"e":60,"code":["- a parallel fan-out group with a join barrier."]},"column":{"s":2,"e":47}},"dim":["","list.14","listItem.1","paragraph.0"],"code":"a parallel fan-out group with a join barrier."},{"id":"/root/children/14/children/1/children/0/children/0","type":"text","loc":{"start":1459,"end":1504,"line":{"s":60,"e":60,"code":["- a parallel fan-out group with a join barrier."]},"column":{"s":2,"e":47}},"dim":["","list.14","listItem.1","paragraph.0","text.0"],"code":"a parallel fan-out group with a join barrier."},{"id":"/root/children/15","type":"paragraph","loc":{"start":1506,"end":1556,"line":{"s":62,"e":62,"code":["Orchestrated Workflow with Barrier Synchronization"]},"column":{"s":0,"e":50}},"dim":["","paragraph.15"],"code":"Orchestrated Workflow with Barrier Synchronization"},{"id":"/root/children/15/children/0","type":"text","loc":{"start":1506,"end":1556,"line":{"s":62,"e":62,"code":["Orchestrated Workflow with Barrier Synchronization"]},"column":{"s":0,"e":50}},"dim":["","paragraph.15","text.0"],"code":"Orchestrated Workflow with Barrier Synchronization"},{"id":"/root/children/16","type":"list","loc":{"start":1558,"end":1827,"line":{"s":64,"e":68,"code":["- Reads the workflow definition (your array).","- For each step:","- If it’s a single task → publish to a serial queue, wait for ack.","- If it’s a parallel block → fan-out messages to a parallel queue,"," track N acks, wait until all are done (barrier),continue to next step."]},"column":{"s":0,"e":72}},"dim":["","list.16"],"code":"- Reads the workflow definition (your array).\n- For each step:\n- If it’s a single task → publish to a serial queue, wait for ack.\n- If it’s a parallel block → fan-out messages to a parallel queue,\n track N acks, wait until all are done (barrier),continue to next step.","symbName":"list","symbRange":[1829,1862],"symbRangeL":[64,71],"outerCode":"- For each step:\n- If it’s a single task → publish to a serial queue, wait for ack.\n- If it’s a parallel block → fan-out messages to a parallel queue,\n track N acks, wait until all are done (barrier),continue to next step.\n\n## problem: wait for bag of tasks","outerHtml":"\n\n

problem: wait for bag of tasks

"},{"id":"/root/children/16/children/0","type":"listItem","loc":{"start":1558,"end":1603,"line":{"s":64,"e":64,"code":["- Reads the workflow definition (your array)."]},"column":{"s":0,"e":45}},"dim":["","list.16","listItem.0"],"code":"- Reads the workflow definition (your array)."},{"id":"/root/children/16/children/0/children/0","type":"paragraph","loc":{"start":1560,"end":1603,"line":{"s":64,"e":64,"code":["- Reads the workflow definition (your array)."]},"column":{"s":2,"e":45}},"dim":["","list.16","listItem.0","paragraph.0"],"code":"Reads the workflow definition (your array)."},{"id":"/root/children/16/children/0/children/0/children/0","type":"text","loc":{"start":1560,"end":1603,"line":{"s":64,"e":64,"code":["- Reads the workflow definition (your array)."]},"column":{"s":2,"e":45}},"dim":["","list.16","listItem.0","paragraph.0","text.0"],"code":"Reads the workflow definition (your array)."},{"id":"/root/children/16/children/1","type":"listItem","loc":{"start":1604,"end":1620,"line":{"s":65,"e":65,"code":["- For each step:"]},"column":{"s":0,"e":16}},"dim":["","list.16","listItem.1"],"code":"- For each step:"},{"id":"/root/children/16/children/1/children/0","type":"paragraph","loc":{"start":1606,"end":1620,"line":{"s":65,"e":65,"code":["- For each step:"]},"column":{"s":2,"e":16}},"dim":["","list.16","listItem.1","paragraph.0"],"code":"For each step:"},{"id":"/root/children/16/children/1/children/0/children/0","type":"text","loc":{"start":1606,"end":1620,"line":{"s":65,"e":65,"code":["- For each step:"]},"column":{"s":2,"e":16}},"dim":["","list.16","listItem.1","paragraph.0","text.0"],"code":"For each step:"},{"id":"/root/children/16/children/2","type":"listItem","loc":{"start":1621,"end":1687,"line":{"s":66,"e":66,"code":["- If it’s a single task → publish to a serial queue, wait for ack."]},"column":{"s":0,"e":66}},"dim":["","list.16","listItem.2"],"code":"- If it’s a single task → publish to a serial queue, wait for ack."},{"id":"/root/children/16/children/2/children/0","type":"paragraph","loc":{"start":1623,"end":1687,"line":{"s":66,"e":66,"code":["- If it’s a single task → publish to a serial queue, wait for ack."]},"column":{"s":2,"e":66}},"dim":["","list.16","listItem.2","paragraph.0"],"code":"If it’s a single task → publish to a serial queue, wait for ack."},{"id":"/root/children/16/children/2/children/0/children/0","type":"text","loc":{"start":1623,"end":1687,"line":{"s":66,"e":66,"code":["- If it’s a single task → publish to a serial queue, wait for ack."]},"column":{"s":2,"e":66}},"dim":["","list.16","listItem.2","paragraph.0","text.0"],"code":"If it’s a single task → publish to a serial queue, wait for ack."},{"id":"/root/children/16/children/3","type":"listItem","loc":{"start":1688,"end":1827,"line":{"s":67,"e":68,"code":["- If it’s a parallel block → fan-out messages to a parallel queue,"," track N acks, wait until all are done (barrier),continue to next step."]},"column":{"s":0,"e":72}},"dim":["","list.16","listItem.3"],"code":"- If it’s a parallel block → fan-out messages to a parallel queue,\n track N acks, wait until all are done (barrier),continue to next step."},{"id":"/root/children/16/children/3/children/0","type":"paragraph","loc":{"start":1690,"end":1827,"line":{"s":67,"e":68,"code":["- If it’s a parallel block → fan-out messages to a parallel queue,"," track N acks, wait until all are done (barrier),continue to next step."]},"column":{"s":2,"e":72}},"dim":["","list.16","listItem.3","paragraph.0"],"code":"If it’s a parallel block → fan-out messages to a parallel queue,\n track N acks, wait until all are done (barrier),continue to next step."},{"id":"/root/children/16/children/3/children/0/children/0","type":"text","loc":{"start":1690,"end":1827,"line":{"s":67,"e":68,"code":["- If it’s a parallel block → fan-out messages to a parallel queue,"," track N acks, wait until all are done (barrier),continue to next step."]},"column":{"s":2,"e":72}},"dim":["","list.16","listItem.3","paragraph.0","text.0"],"code":"If it’s a parallel block → fan-out messages to a parallel queue,\n track N acks, wait until all are done (barrier),continue to next step."},{"id":"/root/children/17","type":"heading","loc":{"start":1829,"end":1862,"line":{"s":70,"e":70,"code":["## problem: wait for bag of tasks"]},"column":{"s":0,"e":33}},"dim":["","heading.17"],"code":"## problem: wait for bag of tasks","symbName":"heading","symbRange":[1864,2009],"symbRangeL":[70,76],"outerCode":"\n- its called Barrier of orchestration\n- or logical cut\n- its how to wait all parallel tasks have finished, and we shall proceed with\n next tasks","outerHtml":"\n"},{"id":"/root/children/17/children/0","type":"text","loc":{"start":1832,"end":1862,"line":{"s":70,"e":70,"code":["## problem: wait for bag of tasks"]},"column":{"s":3,"e":33}},"dim":["","heading.17","text.0"],"code":"problem: wait for bag of tasks"},{"id":"/root/children/18","type":"list","loc":{"start":1864,"end":2009,"line":{"s":72,"e":75,"code":["- its called Barrier of orchestration","- or logical cut","- its how to wait all parallel tasks have finished, and we shall proceed with"," next tasks"]},"column":{"s":0,"e":12}},"dim":["","list.18"],"code":"- its called Barrier of orchestration\n- or logical cut\n- its how to wait all parallel tasks have finished, and we shall proceed with\n next tasks","symbName":"list","symbRange":[2011,2041],"symbRangeL":[72,78],"outerCode":"- or logical cut\n- its how to wait all parallel tasks have finished, and we shall proceed with\n next tasks\n\n### Redis Pub/Sub notification","outerHtml":"\n\n

Redis Pub/Sub notification

"},{"id":"/root/children/18/children/0","type":"listItem","loc":{"start":1864,"end":1901,"line":{"s":72,"e":72,"code":["- its called Barrier of orchestration"]},"column":{"s":0,"e":37}},"dim":["","list.18","listItem.0"],"code":"- its called Barrier of orchestration"},{"id":"/root/children/18/children/0/children/0","type":"paragraph","loc":{"start":1866,"end":1901,"line":{"s":72,"e":72,"code":["- its called Barrier of orchestration"]},"column":{"s":2,"e":37}},"dim":["","list.18","listItem.0","paragraph.0"],"code":"its called Barrier of orchestration"},{"id":"/root/children/18/children/0/children/0/children/0","type":"text","loc":{"start":1866,"end":1901,"line":{"s":72,"e":72,"code":["- its called Barrier of orchestration"]},"column":{"s":2,"e":37}},"dim":["","list.18","listItem.0","paragraph.0","text.0"],"code":"its called Barrier of orchestration"},{"id":"/root/children/18/children/1","type":"listItem","loc":{"start":1902,"end":1918,"line":{"s":73,"e":73,"code":["- or logical cut"]},"column":{"s":0,"e":16}},"dim":["","list.18","listItem.1"],"code":"- or logical cut"},{"id":"/root/children/18/children/1/children/0","type":"paragraph","loc":{"start":1904,"end":1918,"line":{"s":73,"e":73,"code":["- or logical cut"]},"column":{"s":2,"e":16}},"dim":["","list.18","listItem.1","paragraph.0"],"code":"or logical cut"},{"id":"/root/children/18/children/1/children/0/children/0","type":"text","loc":{"start":1904,"end":1918,"line":{"s":73,"e":73,"code":["- or logical cut"]},"column":{"s":2,"e":16}},"dim":["","list.18","listItem.1","paragraph.0","text.0"],"code":"or logical cut"},{"id":"/root/children/18/children/2","type":"listItem","loc":{"start":1919,"end":2009,"line":{"s":74,"e":75,"code":["- its how to wait all parallel tasks have finished, and we shall proceed with"," next tasks"]},"column":{"s":0,"e":12}},"dim":["","list.18","listItem.2"],"code":"- its how to wait all parallel tasks have finished, and we shall proceed with\n next tasks"},{"id":"/root/children/18/children/2/children/0","type":"paragraph","loc":{"start":1921,"end":2009,"line":{"s":74,"e":75,"code":["- its how to wait all parallel tasks have finished, and we shall proceed with"," next tasks"]},"column":{"s":2,"e":12}},"dim":["","list.18","listItem.2","paragraph.0"],"code":"its how to wait all parallel tasks have finished, and we shall proceed with\n next tasks"},{"id":"/root/children/18/children/2/children/0/children/0","type":"text","loc":{"start":1921,"end":2009,"line":{"s":74,"e":75,"code":["- its how to wait all parallel tasks have finished, and we shall proceed with"," next tasks"]},"column":{"s":2,"e":12}},"dim":["","list.18","listItem.2","paragraph.0","text.0"],"code":"its how to wait all parallel tasks have finished, and we shall proceed with\n next tasks"},{"id":"/root/children/19","type":"heading","loc":{"start":2011,"end":2041,"line":{"s":77,"e":77,"code":["### Redis Pub/Sub notification"]},"column":{"s":0,"e":30}},"dim":["","heading.19"],"code":"### Redis Pub/Sub notification","symbName":"heading","symbRange":[2043,2508],"symbRangeL":[77,110],"outerCode":"\n- Redis: (best).\n\n- Setup\n\nWhen you publish the parallel tasks:\n\n```\nSet counter:\nSET barrier:: N\n\nWorkers, when done:\n\nDECR barrier::\n\nIf result == 0 →\nPUBLISH barrier:done \":\"\n```\n\nOrchestrator waits:\n\n```\nredisSub.subscribe(\"barrier:done\");\nredisSub.on(\"message\", (channel, msg) => {\n const [wf, step] = msg.split(\":\");\n continueWorkflow(wf, step);\n});\n```\n\n✅ zero polling ✅ instant reaction ✅ very scalable\n\nThis is the cleanest.","outerHtml":"\n
  • Redis: (best).
\n\n
  • Setup
\n\n

When you publish the parallel tasks:

\n\n

```\nSet counter:\nSET barrier:: N

\n\n

Workers, when done:

\n\n

DECR barrier::

\n\n

If result == 0 →\nPUBLISH barrier:done \":\"\n```

\n\n

Orchestrator waits:

\n\n

```\nredisSub.subscribe(\"barrier:done\");\nredisSub.on(\"message\", (channel, msg) => {\n const [wf, step] = msg.split(\":\");\n continueWorkflow(wf, step);\n});\n```

\n\n

✅ zero polling ✅ instant reaction ✅ very scalable

\n\n

This is the cleanest.

"},{"id":"/root/children/19/children/0","type":"text","loc":{"start":2015,"end":2041,"line":{"s":77,"e":77,"code":["### Redis Pub/Sub notification"]},"column":{"s":4,"e":30}},"dim":["","heading.19","text.0"],"code":"Redis Pub/Sub notification"},{"id":"/root/children/20","type":"list","loc":{"start":2043,"end":2068,"line":{"s":79,"e":81,"code":["- Redis: (best).","","- Setup"]},"column":{"s":0,"e":7}},"dim":["","list.20"],"code":"- Redis: (best).\n\n- Setup","symbName":"list","symbRange":[2070,2630],"symbRangeL":[79,118],"outerCode":"\n- Setup\n\nWhen you publish the parallel tasks:\n\n```\nSet counter:\nSET barrier:: N\n\nWorkers, when done:\n\nDECR barrier::\n\nIf result == 0 →\nPUBLISH barrier:done \":\"\n```\n\nOrchestrator waits:\n\n```\nredisSub.subscribe(\"barrier:done\");\nredisSub.on(\"message\", (channel, msg) => {\n const [wf, step] = msg.split(\":\");\n continueWorkflow(wf, step);\n});\n```\n\n✅ zero polling ✅ instant reaction ✅ very scalable\n\nThis is the cleanest.\n\n### join barrier\n\n#### Barrier Counter in Redis (recommended)\n\nSimple, robust, works great with RabbitMQ.\n\nHow it works:","outerHtml":"\n
  • Setup
\n\n

When you publish the parallel tasks:

\n\n

```\nSet counter:\nSET barrier:: N

\n\n

Workers, when done:

\n\n

DECR barrier::

\n\n

If result == 0 →\nPUBLISH barrier:done \":\"\n```

\n\n

Orchestrator waits:

\n\n

```\nredisSub.subscribe(\"barrier:done\");\nredisSub.on(\"message\", (channel, msg) => {\n const [wf, step] = msg.split(\":\");\n continueWorkflow(wf, step);\n});\n```

\n\n

✅ zero polling ✅ instant reaction ✅ very scalable

\n\n

This is the cleanest.

\n\n

join barrier

\n\n

Barrier Counter in Redis (recommended)

\n\n

Simple, robust, works great with RabbitMQ.

\n\n

How it works:

"},{"id":"/root/children/20/children/0","type":"listItem","loc":{"start":2043,"end":2059,"line":{"s":79,"e":79,"code":["- Redis: (best)."]},"column":{"s":0,"e":16}},"dim":["","list.20","listItem.0"],"code":"- Redis: (best)."},{"id":"/root/children/20/children/0/children/0","type":"paragraph","loc":{"start":2045,"end":2059,"line":{"s":79,"e":79,"code":["- Redis: (best)."]},"column":{"s":2,"e":16}},"dim":["","list.20","listItem.0","paragraph.0"],"code":"Redis: (best)."},{"id":"/root/children/20/children/0/children/0/children/0","type":"text","loc":{"start":2045,"end":2059,"line":{"s":79,"e":79,"code":["- Redis: (best)."]},"column":{"s":2,"e":16}},"dim":["","list.20","listItem.0","paragraph.0","text.0"],"code":"Redis: (best)."},{"id":"/root/children/20/children/1","type":"listItem","loc":{"start":2061,"end":2068,"line":{"s":81,"e":81,"code":["- Setup"]},"column":{"s":0,"e":7}},"dim":["","list.20","listItem.1"],"code":"- Setup"},{"id":"/root/children/20/children/1/children/0","type":"paragraph","loc":{"start":2063,"end":2068,"line":{"s":81,"e":81,"code":["- Setup"]},"column":{"s":2,"e":7}},"dim":["","list.20","listItem.1","paragraph.0"],"code":"Setup"},{"id":"/root/children/20/children/1/children/0/children/0","type":"text","loc":{"start":2063,"end":2068,"line":{"s":81,"e":81,"code":["- Setup"]},"column":{"s":2,"e":7}},"dim":["","list.20","listItem.1","paragraph.0","text.0"],"code":"Setup"},{"id":"/root/children/21","type":"paragraph","loc":{"start":2070,"end":2106,"line":{"s":83,"e":83,"code":["When you publish the parallel tasks:"]},"column":{"s":0,"e":36}},"dim":["","paragraph.21"],"code":"When you publish the parallel tasks:"},{"id":"/root/children/21/children/0","type":"text","loc":{"start":2070,"end":2106,"line":{"s":83,"e":83,"code":["When you publish the parallel tasks:"]},"column":{"s":0,"e":36}},"dim":["","paragraph.21","text.0"],"code":"When you publish the parallel tasks:"},{"id":"/root/children/22","type":"code","loc":{"start":2108,"end":2254,"line":{"s":85,"e":95,"code":["```","Set counter:","SET barrier:: N","","Workers, when done:","","DECR barrier::","","If result == 0 →","PUBLISH barrier:done \":\"","```"]},"column":{"s":0,"e":3}},"dim":["","code.22"],"code":"```\nSet counter:\nSET barrier:: N\n\nWorkers, when done:\n\nDECR barrier::\n\nIf result == 0 →\nPUBLISH barrier:done \":\"\n```","symbName":"code","symbRange":[2256,2275],"symbRangeL":[null,98],"outerCode":";{ engine: dot, rankdir: TD, applyCss: dark, useGvPack: true, clusterize: false, intention: full }\n\n# pattern: orchestrator\n\n- uses 3 queues\n- structure: serial, exclusive+parallel, serial\n- example:\n - `[ serial tasks ]`\n - `[ bag of parallel tasks ]`\n - `[ untiloop ]`\n\n## experimental\n\n- TODO: could also support eval `code` ?\n- holds program state in namespaced redis keys: flat json\n- state can collect output of tasks in adjacent structure\n\n## Recommended minimal clean setup (3 queues)\n\n- /queue/orchestrator\n - 100% serial queue: prefetch = 1, 1 consumer only\n - State machine messages only (continue workflow, step done, parallel done,\n start workflow, etc.)\n\n- /queue/serial-tasks\n - 100% serial queue: prefetch = 1, 1 consumer only\n - tasks run sequentially by nature\n\n- /queue/parallel-tasks\n - many consumers\n - tasks run concurrently\n\n- each task updates its state after finished\n - TODO: but how?\n - completion message\n\nThis is the simplest clean design that gives you:\n\n- controlled sequential execution\n- unlimited parallelism\n- simple, readable routing\n- non-interfering logic\n\n### queues\n\n- queue_orchestrator\n- queue_tasks_serial\n- queue_tasks_parallel\n\n## Workflow Orchestration\n\nRabbitMQ is only the transport; the orchestration logic lives in\nthe program-json.\n\nThis is a “Workflow Orchestration” (a.k.a.\nSaga Orchestration) pattern, a classic Saga orchestration where every “step”\nis either:\n\n- a sequential action, or\n- a parallel fan-out group with a join barrier.\n\nOrchestrated Workflow with Barrier Synchronization\n\n- Reads the workflow definition (your array).\n- For each step:\n- If it’s a single task → publish to a serial queue, wait for ack.\n- If it’s a parallel block → fan-out messages to a parallel queue,\n track N acks, wait until all are done (barrier),continue to next step.\n\n## problem: wait for bag of tasks\n\n- its called Barrier of orchestration\n- or logical cut\n- its how to wait all parallel tasks have finished, and we shall proceed with\n next tasks\n\n### Redis Pub/Sub notification\n\n- Redis: (best).\n\n- Setup\n\nWhen you publish the parallel tasks:\n\n```\nSet counter:\nSET barrier:: N\n\nWorkers, when done:\n\nDECR barrier::\n\nIf result == 0 →\nPUBLISH barrier:done \":\"\n```\n\nOrchestrator waits:","outerHtml":"

;{ engine: dot, rankdir: TD, applyCss: dark, useGvPack: true, clusterize: false, intention: full }

\n\n

pattern: orchestrator

\n\n
  • uses 3 queues
  • structure: serial, exclusive+parallel, serial
  • example:
    • `[ serial tasks ]`
    • `[ bag of parallel tasks ]`
    • `[ untiloop ]`
\n\n

experimental

\n\n
  • TODO: could also support eval `code` ?
  • holds program state in namespaced redis keys: flat json
  • state can collect output of tasks in adjacent structure
\n\n

Recommended minimal clean setup (3 queues)

\n\n
  • /queue/orchestrator
    • 100% serial queue: prefetch = 1, 1 consumer only
    • State machine messages only (continue workflow, step done, parallel done,\n start workflow, etc.)
\n\n
  • /queue/serial-tasks
    • 100% serial queue: prefetch = 1, 1 consumer only
    • tasks run sequentially by nature
\n\n
  • /queue/parallel-tasks
    • many consumers
    • tasks run concurrently
\n\n
  • each task updates its state after finished
    • TODO: but how?
    • completion message
\n\n

This is the simplest clean design that gives you:

\n\n
  • controlled sequential execution
  • unlimited parallelism
  • simple, readable routing
  • non-interfering logic
\n\n

queues

\n\n
  • queue_orchestrator
  • queue_tasks_serial
  • queue_tasks_parallel
\n\n

Workflow Orchestration

\n\n

RabbitMQ is only the transport; the orchestration logic lives in\nthe program-json.

\n\n

This is a “Workflow Orchestration” (a.k.a.\nSaga Orchestration) pattern, a classic Saga orchestration where every “step”\nis either:

\n\n
  • a sequential action, or
  • a parallel fan-out group with a join barrier.
\n\n

Orchestrated Workflow with Barrier Synchronization

\n\n
  • Reads the workflow definition (your array).
  • For each step:
  • If it’s a single task → publish to a serial queue, wait for ack.
  • If it’s a parallel block → fan-out messages to a parallel queue,\n track N acks, wait until all are done (barrier),continue to next step.
\n\n

problem: wait for bag of tasks

\n\n
  • its called Barrier of orchestration
  • or logical cut
  • its how to wait all parallel tasks have finished, and we shall proceed with\n next tasks
\n\n

Redis Pub/Sub notification

\n\n
  • Redis: (best).
\n\n
  • Setup
\n\n

When you publish the parallel tasks:

\n\n

```\nSet counter:\nSET barrier:: N

\n\n

Workers, when done:

\n\n

DECR barrier::

\n\n

If result == 0 →\nPUBLISH barrier:done \":\"\n```

\n\n

Orchestrator waits:

"},{"id":"/root/children/23","type":"paragraph","loc":{"start":2256,"end":2275,"line":{"s":97,"e":97,"code":["Orchestrator waits:"]},"column":{"s":0,"e":19}},"dim":["","paragraph.23"],"code":"Orchestrator waits:"},{"id":"/root/children/23/children/0","type":"text","loc":{"start":2256,"end":2275,"line":{"s":97,"e":97,"code":["Orchestrator waits:"]},"column":{"s":0,"e":19}},"dim":["","paragraph.23","text.0"],"code":"Orchestrator waits:"},{"id":"/root/children/24","type":"code","loc":{"start":2277,"end":2434,"line":{"s":99,"e":105,"code":["```","redisSub.subscribe(\"barrier:done\");","redisSub.on(\"message\", (channel, msg) => {"," const [wf, step] = msg.split(\":\");"," continueWorkflow(wf, step);","});","```"]},"column":{"s":0,"e":3}},"dim":["","code.24"],"code":"```\nredisSub.subscribe(\"barrier:done\");\nredisSub.on(\"message\", (channel, msg) => {\n const [wf, step] = msg.split(\":\");\n continueWorkflow(wf, step);\n});\n```","symbName":"code","symbRange":[2436,4698],"symbRangeL":[null,197],"outerCode":";{ engine: dot, rankdir: TD, applyCss: dark, useGvPack: true, clusterize: false, intention: full }\n\n# pattern: orchestrator\n\n- uses 3 queues\n- structure: serial, exclusive+parallel, serial\n- example:\n - `[ serial tasks ]`\n - `[ bag of parallel tasks ]`\n - `[ untiloop ]`\n\n## experimental\n\n- TODO: could also support eval `code` ?\n- holds program state in namespaced redis keys: flat json\n- state can collect output of tasks in adjacent structure\n\n## Recommended minimal clean setup (3 queues)\n\n- /queue/orchestrator\n - 100% serial queue: prefetch = 1, 1 consumer only\n - State machine messages only (continue workflow, step done, parallel done,\n start workflow, etc.)\n\n- /queue/serial-tasks\n - 100% serial queue: prefetch = 1, 1 consumer only\n - tasks run sequentially by nature\n\n- /queue/parallel-tasks\n - many consumers\n - tasks run concurrently\n\n- each task updates its state after finished\n - TODO: but how?\n - completion message\n\nThis is the simplest clean design that gives you:\n\n- controlled sequential execution\n- unlimited parallelism\n- simple, readable routing\n- non-interfering logic\n\n### queues\n\n- queue_orchestrator\n- queue_tasks_serial\n- queue_tasks_parallel\n\n## Workflow Orchestration\n\nRabbitMQ is only the transport; the orchestration logic lives in\nthe program-json.\n\nThis is a “Workflow Orchestration” (a.k.a.\nSaga Orchestration) pattern, a classic Saga orchestration where every “step”\nis either:\n\n- a sequential action, or\n- a parallel fan-out group with a join barrier.\n\nOrchestrated Workflow with Barrier Synchronization\n\n- Reads the workflow definition (your array).\n- For each step:\n- If it’s a single task → publish to a serial queue, wait for ack.\n- If it’s a parallel block → fan-out messages to a parallel queue,\n track N acks, wait until all are done (barrier),continue to next step.\n\n## problem: wait for bag of tasks\n\n- its called Barrier of orchestration\n- or logical cut\n- its how to wait all parallel tasks have finished, and we shall proceed with\n next tasks\n\n### Redis Pub/Sub notification\n\n- Redis: (best).\n\n- Setup\n\nWhen you publish the parallel tasks:\n\n```\nSet counter:\nSET barrier:: N\n\nWorkers, when done:\n\nDECR barrier::\n\nIf result == 0 →\nPUBLISH barrier:done \":\"\n```\n\nOrchestrator waits:\n\n```\nredisSub.subscribe(\"barrier:done\");\nredisSub.on(\"message\", (channel, msg) => {\n const [wf, step] = msg.split(\":\");\n continueWorkflow(wf, step);\n});\n```\n\n✅ zero polling ✅ instant reaction ✅ very scalable\n\nThis is the cleanest.\n\n### join barrier\n\n#### Barrier Counter in Redis (recommended)\n\nSimple, robust, works great with RabbitMQ.\n\nHow it works:\n\n- Before publishing N parallel tasks → SET barrier::pending = N\n- Each worker, after finishing, runs: DECR barrier::pending\n- When the value hits 0, the orchestrator continues to next step.\n\nWhy this is best:\n\n- Atomic\n- No race conditions\n- Survives crashes\n\nVery easy in Node.js + STOMP\n\n#### Barrier via “completion messages” queue\n\nRabbitMQ-only solution. Pattern:\n\n- Each parallel task sends a completion message to a dedicated queue:\n `workflow..step..completed`\n- Orchestrator counts how many \"completed\" messages arrived.\n- When it reaches N → continue.\n\nCons:\n\n- Requires the orchestrator to stay online and track counts in memory or\n storage.\n- Not safe without persistent storage.\n- Use only when you cannot add Redis/Postgres.\n\n## problem: parallel programs?\n\n- can multiple programs run in the same mq-space?\n- will a slow program not hold up a fast one?\n- possible solution: sharding into few queue's\n\nYou said:\n\n- fast workflow: occurs often\n- slow workflow: occurs rarely\n- slow workflow takes long time\n\nYou want maximum simplicity, so:\n\n✅ Choose Option A: multiple orchestrator queues\n\nYou only need two:\n\n/queue/orchestrator.fast /queue/orchestrator.slow\n\nThen route based on workflow type:\n\nconst destination = workflow.type === 'fast' ?\n'/queue/orchestrator.fast' : '/queue/orchestrator.slow'\n\nEach queue has its own orchestrator consumer.\n\n✅ Fast workflows never wait behind slow ones ✅ No complexity ✅ No risk ✅\nPerfect isolation\n\n### sharding\n\nMultiple orchestrator queues, each one handles many workflows, but\neach workflow stays in exactly one shard based on\nhash(workflowId)% N. Gives you isolation, parallelism, and ordering with\nbounded queue count.\n\n## conclusion\n\n- we use redis. is redisJSON needed? no.\n- main queue: queue_orchestrator\n- no single long-running program\n- program is executed in steps, steps are a disconnected sequence of messages\n\n### state\n\n- central state of the program\n- who holds the program?\n - one big redis json `program:id`?\n - or flat scalar json namespaced like folders:","outerHtml":"

;{ engine: dot, rankdir: TD, applyCss: dark, useGvPack: true, clusterize: false, intention: full }

\n\n

pattern: orchestrator

\n\n
  • uses 3 queues
  • structure: serial, exclusive+parallel, serial
  • example:
    • `[ serial tasks ]`
    • `[ bag of parallel tasks ]`
    • `[ untiloop ]`
\n\n

experimental

\n\n
  • TODO: could also support eval `code` ?
  • holds program state in namespaced redis keys: flat json
  • state can collect output of tasks in adjacent structure
\n\n

Recommended minimal clean setup (3 queues)

\n\n
  • /queue/orchestrator
    • 100% serial queue: prefetch = 1, 1 consumer only
    • State machine messages only (continue workflow, step done, parallel done,\n start workflow, etc.)
\n\n
  • /queue/serial-tasks
    • 100% serial queue: prefetch = 1, 1 consumer only
    • tasks run sequentially by nature
\n\n
  • /queue/parallel-tasks
    • many consumers
    • tasks run concurrently
\n\n
  • each task updates its state after finished
    • TODO: but how?
    • completion message
\n\n

This is the simplest clean design that gives you:

\n\n
  • controlled sequential execution
  • unlimited parallelism
  • simple, readable routing
  • non-interfering logic
\n\n

queues

\n\n
  • queue_orchestrator
  • queue_tasks_serial
  • queue_tasks_parallel
\n\n

Workflow Orchestration

\n\n

RabbitMQ is only the transport; the orchestration logic lives in\nthe program-json.

\n\n

This is a “Workflow Orchestration” (a.k.a.\nSaga Orchestration) pattern, a classic Saga orchestration where every “step”\nis either:

\n\n
  • a sequential action, or
  • a parallel fan-out group with a join barrier.
\n\n

Orchestrated Workflow with Barrier Synchronization

\n\n
  • Reads the workflow definition (your array).
  • For each step:
  • If it’s a single task → publish to a serial queue, wait for ack.
  • If it’s a parallel block → fan-out messages to a parallel queue,\n track N acks, wait until all are done (barrier),continue to next step.
\n\n

problem: wait for bag of tasks

\n\n
  • its called Barrier of orchestration
  • or logical cut
  • its how to wait all parallel tasks have finished, and we shall proceed with\n next tasks
\n\n

Redis Pub/Sub notification

\n\n
  • Redis: (best).
\n\n
  • Setup
\n\n

When you publish the parallel tasks:

\n\n

```\nSet counter:\nSET barrier:: N

\n\n

Workers, when done:

\n\n

DECR barrier::

\n\n

If result == 0 →\nPUBLISH barrier:done \":\"\n```

\n\n

Orchestrator waits:

\n\n

```\nredisSub.subscribe(\"barrier:done\");\nredisSub.on(\"message\", (channel, msg) => {\n const [wf, step] = msg.split(\":\");\n continueWorkflow(wf, step);\n});\n```

\n\n

✅ zero polling ✅ instant reaction ✅ very scalable

\n\n

This is the cleanest.

\n\n

join barrier

\n\n

Barrier Counter in Redis (recommended)

\n\n

Simple, robust, works great with RabbitMQ.

\n\n

How it works:

\n\n
  • Before publishing N parallel tasks → SET barrier::pending = N
  • Each worker, after finishing, runs: DECR barrier::pending
  • When the value hits 0, the orchestrator continues to next step.
\n\n

Why this is best:

\n\n
  • Atomic
  • No race conditions
  • Survives crashes
\n\n

Very easy in Node.js + STOMP

\n\n

Barrier via “completion messages” queue

\n\n

RabbitMQ-only solution. Pattern:

\n\n
  • Each parallel task sends a completion message to a dedicated queue:\n `workflow..step..completed`
  • Orchestrator counts how many \"completed\" messages arrived.
  • When it reaches N → continue.
\n\n

Cons:

\n\n
  • Requires the orchestrator to stay online and track counts in memory or\n storage.
  • Not safe without persistent storage.
  • Use only when you cannot add Redis/Postgres.
\n\n

problem: parallel programs?

\n\n
  • can multiple programs run in the same mq-space?
  • will a slow program not hold up a fast one?
  • possible solution: sharding into few queue's
\n\n

You said:

\n\n
  • fast workflow: occurs often
  • slow workflow: occurs rarely
  • slow workflow takes long time
\n\n

You want maximum simplicity, so:

\n\n

✅ Choose Option A: multiple orchestrator queues

\n\n

You only need two:

\n\n

/queue/orchestrator.fast /queue/orchestrator.slow

\n\n

Then route based on workflow type:

\n\n

const destination = workflow.type === 'fast' ?\n'/queue/orchestrator.fast' : '/queue/orchestrator.slow'

\n\n

Each queue has its own orchestrator consumer.

\n\n

✅ Fast workflows never wait behind slow ones ✅ No complexity ✅ No risk ✅\nPerfect isolation

\n\n

sharding

\n\n

Multiple orchestrator queues, each one handles many workflows, but\neach workflow stays in exactly one shard based on\nhash(workflowId)% N. Gives you isolation, parallelism, and ordering with\nbounded queue count.

\n\n

conclusion

\n\n
  • we use redis. is redisJSON needed? no.
  • main queue: queue_orchestrator
  • no single long-running program
  • program is executed in steps, steps are a disconnected sequence of messages
\n\n

state

\n\n
  • central state of the program
  • who holds the program?
    • one big redis json `program:id`?
    • or flat scalar json namespaced like folders:
"},{"id":"/root/children/25","type":"paragraph","loc":{"start":2436,"end":2485,"line":{"s":107,"e":107,"code":["✅ zero polling ✅ instant reaction ✅ very scalable"]},"column":{"s":0,"e":49}},"dim":["","paragraph.25"],"code":"✅ zero polling ✅ instant reaction ✅ very scalable"},{"id":"/root/children/25/children/0","type":"text","loc":{"start":2436,"end":2485,"line":{"s":107,"e":107,"code":["✅ zero polling ✅ instant reaction ✅ very scalable"]},"column":{"s":0,"e":49}},"dim":["","paragraph.25","text.0"],"code":"✅ zero polling ✅ instant reaction ✅ very scalable"},{"id":"/root/children/26","type":"paragraph","loc":{"start":2487,"end":2508,"line":{"s":109,"e":109,"code":["This is the cleanest."]},"column":{"s":0,"e":21}},"dim":["","paragraph.26"],"code":"This is the cleanest."},{"id":"/root/children/26/children/0","type":"text","loc":{"start":2487,"end":2508,"line":{"s":109,"e":109,"code":["This is the cleanest."]},"column":{"s":0,"e":21}},"dim":["","paragraph.26","text.0"],"code":"This is the cleanest."},{"id":"/root/children/27","type":"heading","loc":{"start":2510,"end":2526,"line":{"s":111,"e":111,"code":["### join barrier"]},"column":{"s":0,"e":16}},"dim":["","heading.27"],"code":"### join barrier","symbName":"heading","symbRange":[2528,5317],"symbRangeL":[111,112],"outerCode":"","outerHtml":""},{"id":"/root/children/27/children/0","type":"text","loc":{"start":2514,"end":2526,"line":{"s":111,"e":111,"code":["### join barrier"]},"column":{"s":4,"e":16}},"dim":["","heading.27","text.0"],"code":"join barrier"},{"id":"/root/children/28","type":"heading","loc":{"start":2528,"end":2571,"line":{"s":113,"e":113,"code":["#### Barrier Counter in Redis (recommended)"]},"column":{"s":0,"e":43}},"dim":["","heading.28"],"code":"#### Barrier Counter in Redis (recommended)","symbName":"heading","symbRange":[2573,2944],"symbRangeL":[113,130],"outerCode":"\nSimple, robust, works great with RabbitMQ.\n\nHow it works:\n\n- Before publishing N parallel tasks → SET barrier::pending = N\n- Each worker, after finishing, runs: DECR barrier::pending\n- When the value hits 0, the orchestrator continues to next step.\n\nWhy this is best:\n\n- Atomic\n- No race conditions\n- Survives crashes\n\nVery easy in Node.js + STOMP","outerHtml":"\n

Simple, robust, works great with RabbitMQ.

\n\n

How it works:

\n\n
  • Before publishing N parallel tasks → SET barrier::pending = N
  • Each worker, after finishing, runs: DECR barrier::pending
  • When the value hits 0, the orchestrator continues to next step.
\n\n

Why this is best:

\n\n
  • Atomic
  • No race conditions
  • Survives crashes
\n\n

Very easy in Node.js + STOMP

"},{"id":"/root/children/28/children/0","type":"text","loc":{"start":2533,"end":2571,"line":{"s":113,"e":113,"code":["#### Barrier Counter in Redis (recommended)"]},"column":{"s":5,"e":43}},"dim":["","heading.28","text.0"],"code":"Barrier Counter in Redis (recommended)"},{"id":"/root/children/29","type":"paragraph","loc":{"start":2573,"end":2615,"line":{"s":115,"e":115,"code":["Simple, robust, works great with RabbitMQ."]},"column":{"s":0,"e":42}},"dim":["","paragraph.29"],"code":"Simple, robust, works great with RabbitMQ."},{"id":"/root/children/29/children/0","type":"text","loc":{"start":2573,"end":2615,"line":{"s":115,"e":115,"code":["Simple, robust, works great with RabbitMQ."]},"column":{"s":0,"e":42}},"dim":["","paragraph.29","text.0"],"code":"Simple, robust, works great with RabbitMQ."},{"id":"/root/children/30","type":"paragraph","loc":{"start":2617,"end":2630,"line":{"s":117,"e":117,"code":["How it works:"]},"column":{"s":0,"e":13}},"dim":["","paragraph.30"],"code":"How it works:"},{"id":"/root/children/30/children/0","type":"text","loc":{"start":2617,"end":2630,"line":{"s":117,"e":117,"code":["How it works:"]},"column":{"s":0,"e":13}},"dim":["","paragraph.30","text.0"],"code":"How it works:"},{"id":"/root/children/31","type":"list","loc":{"start":2632,"end":2845,"line":{"s":119,"e":121,"code":["- Before publishing N parallel tasks → SET barrier::pending = N","- Each worker, after finishing, runs: DECR barrier::pending","- When the value hits 0, the orchestrator continues to next step."]},"column":{"s":0,"e":65}},"dim":["","list.31"],"code":"- Before publishing N parallel tasks → SET barrier::pending = N\n- Each worker, after finishing, runs: DECR barrier::pending\n- When the value hits 0, the orchestrator continues to next step.","symbName":"list","symbRange":[2847,2864],"symbRangeL":[119,124],"outerCode":"- Each worker, after finishing, runs: DECR barrier::pending\n- When the value hits 0, the orchestrator continues to next step.\n\nWhy this is best:","outerHtml":"
  • Each worker, after finishing, runs: DECR barrier::pending
  • When the value hits 0, the orchestrator continues to next step.
\n\n

Why this is best:

"},{"id":"/root/children/31/children/0","type":"listItem","loc":{"start":2632,"end":2707,"line":{"s":119,"e":119,"code":["- Before publishing N parallel tasks → SET barrier::pending = N"]},"column":{"s":0,"e":75}},"dim":["","list.31","listItem.0"],"code":"- Before publishing N parallel tasks → SET barrier::pending = N"},{"id":"/root/children/31/children/0/children/0","type":"paragraph","loc":{"start":2634,"end":2707,"line":{"s":119,"e":119,"code":["- Before publishing N parallel tasks → SET barrier::pending = N"]},"column":{"s":2,"e":75}},"dim":["","list.31","listItem.0","paragraph.0"],"code":"Before publishing N parallel tasks → SET barrier::pending = N"},{"id":"/root/children/31/children/0/children/0/children/0","type":"text","loc":{"start":2634,"end":2683,"line":{"s":119,"e":119,"code":["- Before publishing N parallel tasks → SET barrier::pending = N"]},"column":{"s":2,"e":51}},"dim":["","list.31","listItem.0","paragraph.0","text.0"],"code":"Before publishing N parallel tasks → SET barrier:"},{"id":"/root/children/31/children/0/children/0/children/1","type":"html","loc":{"start":2683,"end":2695,"line":{"s":119,"e":119,"code":["- Before publishing N parallel tasks → SET barrier::pending = N"]},"column":{"s":51,"e":63}},"dim":["","list.31","listItem.0","paragraph.0","html.1"],"code":""},{"id":"/root/children/31/children/0/children/0/children/2","type":"text","loc":{"start":2695,"end":2707,"line":{"s":119,"e":119,"code":["- Before publishing N parallel tasks → SET barrier::pending = N"]},"column":{"s":63,"e":75}},"dim":["","list.31","listItem.0","paragraph.0","text.2"],"code":":pending = N"},{"id":"/root/children/31/children/1","type":"listItem","loc":{"start":2708,"end":2779,"line":{"s":120,"e":120,"code":["- Each worker, after finishing, runs: DECR barrier::pending"]},"column":{"s":0,"e":71}},"dim":["","list.31","listItem.1"],"code":"- Each worker, after finishing, runs: DECR barrier::pending"},{"id":"/root/children/31/children/1/children/0","type":"paragraph","loc":{"start":2710,"end":2779,"line":{"s":120,"e":120,"code":["- Each worker, after finishing, runs: DECR barrier::pending"]},"column":{"s":2,"e":71}},"dim":["","list.31","listItem.1","paragraph.0"],"code":"Each worker, after finishing, runs: DECR barrier::pending"},{"id":"/root/children/31/children/1/children/0/children/0","type":"text","loc":{"start":2710,"end":2759,"line":{"s":120,"e":120,"code":["- Each worker, after finishing, runs: DECR barrier::pending"]},"column":{"s":2,"e":51}},"dim":["","list.31","listItem.1","paragraph.0","text.0"],"code":"Each worker, after finishing, runs: DECR barrier:"},{"id":"/root/children/31/children/1/children/0/children/1","type":"html","loc":{"start":2759,"end":2771,"line":{"s":120,"e":120,"code":["- Each worker, after finishing, runs: DECR barrier::pending"]},"column":{"s":51,"e":63}},"dim":["","list.31","listItem.1","paragraph.0","html.1"],"code":""},{"id":"/root/children/31/children/1/children/0/children/2","type":"text","loc":{"start":2771,"end":2779,"line":{"s":120,"e":120,"code":["- Each worker, after finishing, runs: DECR barrier::pending"]},"column":{"s":63,"e":71}},"dim":["","list.31","listItem.1","paragraph.0","text.2"],"code":":pending"},{"id":"/root/children/31/children/2","type":"listItem","loc":{"start":2780,"end":2845,"line":{"s":121,"e":121,"code":["- When the value hits 0, the orchestrator continues to next step."]},"column":{"s":0,"e":65}},"dim":["","list.31","listItem.2"],"code":"- When the value hits 0, the orchestrator continues to next step."},{"id":"/root/children/31/children/2/children/0","type":"paragraph","loc":{"start":2782,"end":2845,"line":{"s":121,"e":121,"code":["- When the value hits 0, the orchestrator continues to next step."]},"column":{"s":2,"e":65}},"dim":["","list.31","listItem.2","paragraph.0"],"code":"When the value hits 0, the orchestrator continues to next step."},{"id":"/root/children/31/children/2/children/0/children/0","type":"text","loc":{"start":2782,"end":2845,"line":{"s":121,"e":121,"code":["- When the value hits 0, the orchestrator continues to next step."]},"column":{"s":2,"e":65}},"dim":["","list.31","listItem.2","paragraph.0","text.0"],"code":"When the value hits 0, the orchestrator continues to next step."},{"id":"/root/children/32","type":"paragraph","loc":{"start":2847,"end":2864,"line":{"s":123,"e":123,"code":["Why this is best:"]},"column":{"s":0,"e":17}},"dim":["","paragraph.32"],"code":"Why this is best:"},{"id":"/root/children/32/children/0","type":"text","loc":{"start":2847,"end":2864,"line":{"s":123,"e":123,"code":["Why this is best:"]},"column":{"s":0,"e":17}},"dim":["","paragraph.32","text.0"],"code":"Why this is best:"},{"id":"/root/children/33","type":"list","loc":{"start":2866,"end":2914,"line":{"s":125,"e":127,"code":["- Atomic","- No race conditions","- Survives crashes"]},"column":{"s":0,"e":18}},"dim":["","list.33"],"code":"- Atomic\n- No race conditions\n- Survives crashes","symbName":"list","symbRange":[2916,3024],"symbRangeL":[125,134],"outerCode":"- No race conditions\n- Survives crashes\n\nVery easy in Node.js + STOMP\n\n#### Barrier via “completion messages” queue\n\nRabbitMQ-only solution. Pattern:","outerHtml":"
  • No race conditions
  • Survives crashes
\n\n

Very easy in Node.js + STOMP

\n\n

Barrier via “completion messages” queue

\n\n

RabbitMQ-only solution. Pattern:

"},{"id":"/root/children/33/children/0","type":"listItem","loc":{"start":2866,"end":2874,"line":{"s":125,"e":125,"code":["- Atomic"]},"column":{"s":0,"e":8}},"dim":["","list.33","listItem.0"],"code":"- Atomic"},{"id":"/root/children/33/children/0/children/0","type":"paragraph","loc":{"start":2868,"end":2874,"line":{"s":125,"e":125,"code":["- Atomic"]},"column":{"s":2,"e":8}},"dim":["","list.33","listItem.0","paragraph.0"],"code":"Atomic"},{"id":"/root/children/33/children/0/children/0/children/0","type":"text","loc":{"start":2868,"end":2874,"line":{"s":125,"e":125,"code":["- Atomic"]},"column":{"s":2,"e":8}},"dim":["","list.33","listItem.0","paragraph.0","text.0"],"code":"Atomic"},{"id":"/root/children/33/children/1","type":"listItem","loc":{"start":2875,"end":2895,"line":{"s":126,"e":126,"code":["- No race conditions"]},"column":{"s":0,"e":20}},"dim":["","list.33","listItem.1"],"code":"- No race conditions"},{"id":"/root/children/33/children/1/children/0","type":"paragraph","loc":{"start":2877,"end":2895,"line":{"s":126,"e":126,"code":["- No race conditions"]},"column":{"s":2,"e":20}},"dim":["","list.33","listItem.1","paragraph.0"],"code":"No race conditions"},{"id":"/root/children/33/children/1/children/0/children/0","type":"text","loc":{"start":2877,"end":2895,"line":{"s":126,"e":126,"code":["- No race conditions"]},"column":{"s":2,"e":20}},"dim":["","list.33","listItem.1","paragraph.0","text.0"],"code":"No race conditions"},{"id":"/root/children/33/children/2","type":"listItem","loc":{"start":2896,"end":2914,"line":{"s":127,"e":127,"code":["- Survives crashes"]},"column":{"s":0,"e":18}},"dim":["","list.33","listItem.2"],"code":"- Survives crashes"},{"id":"/root/children/33/children/2/children/0","type":"paragraph","loc":{"start":2898,"end":2914,"line":{"s":127,"e":127,"code":["- Survives crashes"]},"column":{"s":2,"e":18}},"dim":["","list.33","listItem.2","paragraph.0"],"code":"Survives crashes"},{"id":"/root/children/33/children/2/children/0/children/0","type":"text","loc":{"start":2898,"end":2914,"line":{"s":127,"e":127,"code":["- Survives crashes"]},"column":{"s":2,"e":18}},"dim":["","list.33","listItem.2","paragraph.0","text.0"],"code":"Survives crashes"},{"id":"/root/children/34","type":"paragraph","loc":{"start":2916,"end":2944,"line":{"s":129,"e":129,"code":["Very easy in Node.js + STOMP"]},"column":{"s":0,"e":28}},"dim":["","paragraph.34"],"code":"Very easy in Node.js + STOMP"},{"id":"/root/children/34/children/0","type":"text","loc":{"start":2916,"end":2944,"line":{"s":129,"e":129,"code":["Very easy in Node.js + STOMP"]},"column":{"s":0,"e":28}},"dim":["","paragraph.34","text.0"],"code":"Very easy in Node.js + STOMP"},{"id":"/root/children/35","type":"heading","loc":{"start":2946,"end":2990,"line":{"s":131,"e":131,"code":["#### Barrier via “completion messages” queue"]},"column":{"s":0,"e":44}},"dim":["","heading.35"],"code":"#### Barrier via “completion messages” queue","symbName":"heading","symbRange":[2992,3403],"symbRangeL":[131,146],"outerCode":"\nRabbitMQ-only solution. Pattern:\n\n- Each parallel task sends a completion message to a dedicated queue:\n `workflow..step..completed`\n- Orchestrator counts how many \"completed\" messages arrived.\n- When it reaches N → continue.\n\nCons:\n\n- Requires the orchestrator to stay online and track counts in memory or\n storage.\n- Not safe without persistent storage.\n- Use only when you cannot add Redis/Postgres.","outerHtml":"\n

RabbitMQ-only solution. Pattern:

\n\n
  • Each parallel task sends a completion message to a dedicated queue:\n `workflow..step..completed`
  • Orchestrator counts how many \"completed\" messages arrived.
  • When it reaches N → continue.
\n\n

Cons:

\n\n
  • Requires the orchestrator to stay online and track counts in memory or\n storage.
  • Not safe without persistent storage.
  • Use only when you cannot add Redis/Postgres.
"},{"id":"/root/children/35/children/0","type":"text","loc":{"start":2951,"end":2990,"line":{"s":131,"e":131,"code":["#### Barrier via “completion messages” queue"]},"column":{"s":5,"e":44}},"dim":["","heading.35","text.0"],"code":"Barrier via “completion messages” queue"},{"id":"/root/children/36","type":"paragraph","loc":{"start":2992,"end":3024,"line":{"s":133,"e":133,"code":["RabbitMQ-only solution. Pattern:"]},"column":{"s":0,"e":32}},"dim":["","paragraph.36"],"code":"RabbitMQ-only solution. Pattern:"},{"id":"/root/children/36/children/0","type":"text","loc":{"start":2992,"end":3024,"line":{"s":133,"e":133,"code":["RabbitMQ-only solution. Pattern:"]},"column":{"s":0,"e":32}},"dim":["","paragraph.36","text.0"],"code":"RabbitMQ-only solution. Pattern:"},{"id":"/root/children/37","type":"list","loc":{"start":3026,"end":3225,"line":{"s":135,"e":138,"code":["- Each parallel task sends a completion message to a dedicated queue:"," `workflow..step..completed`","- Orchestrator counts how many \"completed\" messages arrived.","- When it reaches N → continue."]},"column":{"s":0,"e":31}},"dim":["","list.37"],"code":"- Each parallel task sends a completion message to a dedicated queue:\n `workflow..step..completed`\n- Orchestrator counts how many \"completed\" messages arrived.\n- When it reaches N → continue.","symbName":"list","symbRange":[3227,3232],"symbRangeL":[135,141],"outerCode":" `workflow..step..completed`\n- Orchestrator counts how many \"completed\" messages arrived.\n- When it reaches N → continue.\n\nCons:","outerHtml":"

`workflow..step..completed`

  • Orchestrator counts how many \"completed\" messages arrived.
  • When it reaches N → continue.
\n\n

Cons:

"},{"id":"/root/children/37/children/0","type":"listItem","loc":{"start":3026,"end":3132,"line":{"s":135,"e":136,"code":["- Each parallel task sends a completion message to a dedicated queue:"," `workflow..step..completed`"]},"column":{"s":0,"e":36}},"dim":["","list.37","listItem.0"],"code":"- Each parallel task sends a completion message to a dedicated queue:\n `workflow..step..completed`"},{"id":"/root/children/37/children/0/children/0","type":"paragraph","loc":{"start":3028,"end":3132,"line":{"s":135,"e":136,"code":["- Each parallel task sends a completion message to a dedicated queue:"," `workflow..step..completed`"]},"column":{"s":2,"e":36}},"dim":["","list.37","listItem.0","paragraph.0"],"code":"Each parallel task sends a completion message to a dedicated queue:\n `workflow..step..completed`"},{"id":"/root/children/37/children/0/children/0/children/0","type":"text","loc":{"start":3028,"end":3096,"line":{"s":135,"e":136,"code":["- Each parallel task sends a completion message to a dedicated queue:"," `workflow..step..completed`"]},"column":{"s":2,"e":0}},"dim":["","list.37","listItem.0","paragraph.0","text.0"],"code":"Each parallel task sends a completion message to a dedicated queue:\n"},{"id":"/root/children/37/children/0/children/0/children/1","type":"inlineCode","loc":{"start":3098,"end":3132,"line":{"s":136,"e":136,"code":[" `workflow..step..completed`"]},"column":{"s":2,"e":36}},"dim":["","list.37","listItem.0","paragraph.0","inlineCode.1"],"code":"`workflow..step..completed`"},{"id":"/root/children/37/children/1","type":"listItem","loc":{"start":3133,"end":3193,"line":{"s":137,"e":137,"code":["- Orchestrator counts how many \"completed\" messages arrived."]},"column":{"s":0,"e":60}},"dim":["","list.37","listItem.1"],"code":"- Orchestrator counts how many \"completed\" messages arrived."},{"id":"/root/children/37/children/1/children/0","type":"paragraph","loc":{"start":3135,"end":3193,"line":{"s":137,"e":137,"code":["- Orchestrator counts how many \"completed\" messages arrived."]},"column":{"s":2,"e":60}},"dim":["","list.37","listItem.1","paragraph.0"],"code":"Orchestrator counts how many \"completed\" messages arrived."},{"id":"/root/children/37/children/1/children/0/children/0","type":"text","loc":{"start":3135,"end":3193,"line":{"s":137,"e":137,"code":["- Orchestrator counts how many \"completed\" messages arrived."]},"column":{"s":2,"e":60}},"dim":["","list.37","listItem.1","paragraph.0","text.0"],"code":"Orchestrator counts how many \"completed\" messages arrived."},{"id":"/root/children/37/children/2","type":"listItem","loc":{"start":3194,"end":3225,"line":{"s":138,"e":138,"code":["- When it reaches N → continue."]},"column":{"s":0,"e":31}},"dim":["","list.37","listItem.2"],"code":"- When it reaches N → continue."},{"id":"/root/children/37/children/2/children/0","type":"paragraph","loc":{"start":3196,"end":3225,"line":{"s":138,"e":138,"code":["- When it reaches N → continue."]},"column":{"s":2,"e":31}},"dim":["","list.37","listItem.2","paragraph.0"],"code":"When it reaches N → continue."},{"id":"/root/children/37/children/2/children/0/children/0","type":"text","loc":{"start":3196,"end":3225,"line":{"s":138,"e":138,"code":["- When it reaches N → continue."]},"column":{"s":2,"e":31}},"dim":["","list.37","listItem.2","paragraph.0","text.0"],"code":"When it reaches N → continue."},{"id":"/root/children/38","type":"paragraph","loc":{"start":3227,"end":3232,"line":{"s":140,"e":140,"code":["Cons:"]},"column":{"s":0,"e":5}},"dim":["","paragraph.38"],"code":"Cons:"},{"id":"/root/children/38/children/0","type":"text","loc":{"start":3227,"end":3232,"line":{"s":140,"e":140,"code":["Cons:"]},"column":{"s":0,"e":5}},"dim":["","paragraph.38","text.0"],"code":"Cons:"},{"id":"/root/children/39","type":"list","loc":{"start":3234,"end":3403,"line":{"s":142,"e":145,"code":["- Requires the orchestrator to stay online and track counts in memory or"," storage.","- Not safe without persistent storage.","- Use only when you cannot add Redis/Postgres."]},"column":{"s":0,"e":46}},"dim":["","list.39"],"code":"- Requires the orchestrator to stay online and track counts in memory or\n storage.\n- Not safe without persistent storage.\n- Use only when you cannot add Redis/Postgres.","symbName":"list","symbRange":[3405,3435],"symbRangeL":[142,148],"outerCode":" storage.\n- Not safe without persistent storage.\n- Use only when you cannot add Redis/Postgres.\n\n## problem: parallel programs?","outerHtml":"

storage.

  • Not safe without persistent storage.
  • Use only when you cannot add Redis/Postgres.
\n\n

problem: parallel programs?

"},{"id":"/root/children/39/children/0","type":"listItem","loc":{"start":3234,"end":3317,"line":{"s":142,"e":143,"code":["- Requires the orchestrator to stay online and track counts in memory or"," storage."]},"column":{"s":0,"e":10}},"dim":["","list.39","listItem.0"],"code":"- Requires the orchestrator to stay online and track counts in memory or\n storage."},{"id":"/root/children/39/children/0/children/0","type":"paragraph","loc":{"start":3236,"end":3317,"line":{"s":142,"e":143,"code":["- Requires the orchestrator to stay online and track counts in memory or"," storage."]},"column":{"s":2,"e":10}},"dim":["","list.39","listItem.0","paragraph.0"],"code":"Requires the orchestrator to stay online and track counts in memory or\n storage."},{"id":"/root/children/39/children/0/children/0/children/0","type":"text","loc":{"start":3236,"end":3317,"line":{"s":142,"e":143,"code":["- Requires the orchestrator to stay online and track counts in memory or"," storage."]},"column":{"s":2,"e":10}},"dim":["","list.39","listItem.0","paragraph.0","text.0"],"code":"Requires the orchestrator to stay online and track counts in memory or\n storage."},{"id":"/root/children/39/children/1","type":"listItem","loc":{"start":3318,"end":3356,"line":{"s":144,"e":144,"code":["- Not safe without persistent storage."]},"column":{"s":0,"e":38}},"dim":["","list.39","listItem.1"],"code":"- Not safe without persistent storage."},{"id":"/root/children/39/children/1/children/0","type":"paragraph","loc":{"start":3320,"end":3356,"line":{"s":144,"e":144,"code":["- Not safe without persistent storage."]},"column":{"s":2,"e":38}},"dim":["","list.39","listItem.1","paragraph.0"],"code":"Not safe without persistent storage."},{"id":"/root/children/39/children/1/children/0/children/0","type":"text","loc":{"start":3320,"end":3356,"line":{"s":144,"e":144,"code":["- Not safe without persistent storage."]},"column":{"s":2,"e":38}},"dim":["","list.39","listItem.1","paragraph.0","text.0"],"code":"Not safe without persistent storage."},{"id":"/root/children/39/children/2","type":"listItem","loc":{"start":3357,"end":3403,"line":{"s":145,"e":145,"code":["- Use only when you cannot add Redis/Postgres."]},"column":{"s":0,"e":46}},"dim":["","list.39","listItem.2"],"code":"- Use only when you cannot add Redis/Postgres."},{"id":"/root/children/39/children/2/children/0","type":"paragraph","loc":{"start":3359,"end":3403,"line":{"s":145,"e":145,"code":["- Use only when you cannot add Redis/Postgres."]},"column":{"s":2,"e":46}},"dim":["","list.39","listItem.2","paragraph.0"],"code":"Use only when you cannot add Redis/Postgres."},{"id":"/root/children/39/children/2/children/0/children/0","type":"text","loc":{"start":3359,"end":3403,"line":{"s":145,"e":145,"code":["- Use only when you cannot add Redis/Postgres."]},"column":{"s":2,"e":46}},"dim":["","list.39","listItem.2","paragraph.0","text.0"],"code":"Use only when you cannot add Redis/Postgres."},{"id":"/root/children/40","type":"heading","loc":{"start":3405,"end":3435,"line":{"s":147,"e":147,"code":["## problem: parallel programs?"]},"column":{"s":0,"e":30}},"dim":["","heading.40"],"code":"## problem: parallel programs?","symbName":"heading","symbRange":[3437,4117],"symbRangeL":[147,176],"outerCode":"\n- can multiple programs run in the same mq-space?\n- will a slow program not hold up a fast one?\n- possible solution: sharding into few queue's\n\nYou said:\n\n- fast workflow: occurs often\n- slow workflow: occurs rarely\n- slow workflow takes long time\n\nYou want maximum simplicity, so:\n\n✅ Choose Option A: multiple orchestrator queues\n\nYou only need two:\n\n/queue/orchestrator.fast /queue/orchestrator.slow\n\nThen route based on workflow type:\n\nconst destination = workflow.type === 'fast' ?\n'/queue/orchestrator.fast' : '/queue/orchestrator.slow'\n\nEach queue has its own orchestrator consumer.\n\n✅ Fast workflows never wait behind slow ones ✅ No complexity ✅ No risk ✅\nPerfect isolation","outerHtml":"\n
  • can multiple programs run in the same mq-space?
  • will a slow program not hold up a fast one?
  • possible solution: sharding into few queue's
\n\n

You said:

\n\n
  • fast workflow: occurs often
  • slow workflow: occurs rarely
  • slow workflow takes long time
\n\n

You want maximum simplicity, so:

\n\n

✅ Choose Option A: multiple orchestrator queues

\n\n

You only need two:

\n\n

/queue/orchestrator.fast /queue/orchestrator.slow

\n\n

Then route based on workflow type:

\n\n

const destination = workflow.type === 'fast' ?\n'/queue/orchestrator.fast' : '/queue/orchestrator.slow'

\n\n

Each queue has its own orchestrator consumer.

\n\n

✅ Fast workflows never wait behind slow ones ✅ No complexity ✅ No risk ✅\nPerfect isolation

"},{"id":"/root/children/40/children/0","type":"text","loc":{"start":3408,"end":3435,"line":{"s":147,"e":147,"code":["## problem: parallel programs?"]},"column":{"s":3,"e":30}},"dim":["","heading.40","text.0"],"code":"problem: parallel programs?"},{"id":"/root/children/41","type":"list","loc":{"start":3437,"end":3579,"line":{"s":149,"e":151,"code":["- can multiple programs run in the same mq-space?","- will a slow program not hold up a fast one?","- possible solution: sharding into few queue's"]},"column":{"s":0,"e":46}},"dim":["","list.41"],"code":"- can multiple programs run in the same mq-space?\n- will a slow program not hold up a fast one?\n- possible solution: sharding into few queue's","symbName":"list","symbRange":[3581,3590],"symbRangeL":[149,154],"outerCode":"- will a slow program not hold up a fast one?\n- possible solution: sharding into few queue's\n\nYou said:","outerHtml":"
  • will a slow program not hold up a fast one?
  • possible solution: sharding into few queue's
\n\n

You said:

"},{"id":"/root/children/41/children/0","type":"listItem","loc":{"start":3437,"end":3486,"line":{"s":149,"e":149,"code":["- can multiple programs run in the same mq-space?"]},"column":{"s":0,"e":49}},"dim":["","list.41","listItem.0"],"code":"- can multiple programs run in the same mq-space?"},{"id":"/root/children/41/children/0/children/0","type":"paragraph","loc":{"start":3439,"end":3486,"line":{"s":149,"e":149,"code":["- can multiple programs run in the same mq-space?"]},"column":{"s":2,"e":49}},"dim":["","list.41","listItem.0","paragraph.0"],"code":"can multiple programs run in the same mq-space?"},{"id":"/root/children/41/children/0/children/0/children/0","type":"text","loc":{"start":3439,"end":3486,"line":{"s":149,"e":149,"code":["- can multiple programs run in the same mq-space?"]},"column":{"s":2,"e":49}},"dim":["","list.41","listItem.0","paragraph.0","text.0"],"code":"can multiple programs run in the same mq-space?"},{"id":"/root/children/41/children/1","type":"listItem","loc":{"start":3487,"end":3532,"line":{"s":150,"e":150,"code":["- will a slow program not hold up a fast one?"]},"column":{"s":0,"e":45}},"dim":["","list.41","listItem.1"],"code":"- will a slow program not hold up a fast one?"},{"id":"/root/children/41/children/1/children/0","type":"paragraph","loc":{"start":3489,"end":3532,"line":{"s":150,"e":150,"code":["- will a slow program not hold up a fast one?"]},"column":{"s":2,"e":45}},"dim":["","list.41","listItem.1","paragraph.0"],"code":"will a slow program not hold up a fast one?"},{"id":"/root/children/41/children/1/children/0/children/0","type":"text","loc":{"start":3489,"end":3532,"line":{"s":150,"e":150,"code":["- will a slow program not hold up a fast one?"]},"column":{"s":2,"e":45}},"dim":["","list.41","listItem.1","paragraph.0","text.0"],"code":"will a slow program not hold up a fast one?"},{"id":"/root/children/41/children/2","type":"listItem","loc":{"start":3533,"end":3579,"line":{"s":151,"e":151,"code":["- possible solution: sharding into few queue's"]},"column":{"s":0,"e":46}},"dim":["","list.41","listItem.2"],"code":"- possible solution: sharding into few queue's"},{"id":"/root/children/41/children/2/children/0","type":"paragraph","loc":{"start":3535,"end":3579,"line":{"s":151,"e":151,"code":["- possible solution: sharding into few queue's"]},"column":{"s":2,"e":46}},"dim":["","list.41","listItem.2","paragraph.0"],"code":"possible solution: sharding into few queue's"},{"id":"/root/children/41/children/2/children/0/children/0","type":"text","loc":{"start":3535,"end":3579,"line":{"s":151,"e":151,"code":["- possible solution: sharding into few queue's"]},"column":{"s":2,"e":46}},"dim":["","list.41","listItem.2","paragraph.0","text.0"],"code":"possible solution: sharding into few queue's"},{"id":"/root/children/42","type":"paragraph","loc":{"start":3581,"end":3590,"line":{"s":153,"e":153,"code":["You said:"]},"column":{"s":0,"e":9}},"dim":["","paragraph.42"],"code":"You said:"},{"id":"/root/children/42/children/0","type":"text","loc":{"start":3581,"end":3590,"line":{"s":153,"e":153,"code":["You said:"]},"column":{"s":0,"e":9}},"dim":["","paragraph.42","text.0"],"code":"You said:"},{"id":"/root/children/43","type":"list","loc":{"start":3592,"end":3684,"line":{"s":155,"e":157,"code":["- fast workflow: occurs often","- slow workflow: occurs rarely","- slow workflow takes long time"]},"column":{"s":0,"e":31}},"dim":["","list.43"],"code":"- fast workflow: occurs often\n- slow workflow: occurs rarely\n- slow workflow takes long time","symbName":"list","symbRange":[3686,4358],"symbRangeL":[155,185],"outerCode":"- slow workflow: occurs rarely\n- slow workflow takes long time\n\nYou want maximum simplicity, so:\n\n✅ Choose Option A: multiple orchestrator queues\n\nYou only need two:\n\n/queue/orchestrator.fast /queue/orchestrator.slow\n\nThen route based on workflow type:\n\nconst destination = workflow.type === 'fast' ?\n'/queue/orchestrator.fast' : '/queue/orchestrator.slow'\n\nEach queue has its own orchestrator consumer.\n\n✅ Fast workflows never wait behind slow ones ✅ No complexity ✅ No risk ✅\nPerfect isolation\n\n### sharding\n\nMultiple orchestrator queues, each one handles many workflows, but\neach workflow stays in exactly one shard based on\nhash(workflowId)% N. Gives you isolation, parallelism, and ordering with\nbounded queue count.\n\n## conclusion","outerHtml":"
  • slow workflow: occurs rarely
  • slow workflow takes long time
\n\n

You want maximum simplicity, so:

\n\n

✅ Choose Option A: multiple orchestrator queues

\n\n

You only need two:

\n\n

/queue/orchestrator.fast /queue/orchestrator.slow

\n\n

Then route based on workflow type:

\n\n

const destination = workflow.type === 'fast' ?\n'/queue/orchestrator.fast' : '/queue/orchestrator.slow'

\n\n

Each queue has its own orchestrator consumer.

\n\n

✅ Fast workflows never wait behind slow ones ✅ No complexity ✅ No risk ✅\nPerfect isolation

\n\n

sharding

\n\n

Multiple orchestrator queues, each one handles many workflows, but\neach workflow stays in exactly one shard based on\nhash(workflowId)% N. Gives you isolation, parallelism, and ordering with\nbounded queue count.

\n\n

conclusion

"},{"id":"/root/children/43/children/0","type":"listItem","loc":{"start":3592,"end":3621,"line":{"s":155,"e":155,"code":["- fast workflow: occurs often"]},"column":{"s":0,"e":29}},"dim":["","list.43","listItem.0"],"code":"- fast workflow: occurs often"},{"id":"/root/children/43/children/0/children/0","type":"paragraph","loc":{"start":3594,"end":3621,"line":{"s":155,"e":155,"code":["- fast workflow: occurs often"]},"column":{"s":2,"e":29}},"dim":["","list.43","listItem.0","paragraph.0"],"code":"fast workflow: occurs often"},{"id":"/root/children/43/children/0/children/0/children/0","type":"text","loc":{"start":3594,"end":3621,"line":{"s":155,"e":155,"code":["- fast workflow: occurs often"]},"column":{"s":2,"e":29}},"dim":["","list.43","listItem.0","paragraph.0","text.0"],"code":"fast workflow: occurs often"},{"id":"/root/children/43/children/1","type":"listItem","loc":{"start":3622,"end":3652,"line":{"s":156,"e":156,"code":["- slow workflow: occurs rarely"]},"column":{"s":0,"e":30}},"dim":["","list.43","listItem.1"],"code":"- slow workflow: occurs rarely"},{"id":"/root/children/43/children/1/children/0","type":"paragraph","loc":{"start":3624,"end":3652,"line":{"s":156,"e":156,"code":["- slow workflow: occurs rarely"]},"column":{"s":2,"e":30}},"dim":["","list.43","listItem.1","paragraph.0"],"code":"slow workflow: occurs rarely"},{"id":"/root/children/43/children/1/children/0/children/0","type":"text","loc":{"start":3624,"end":3652,"line":{"s":156,"e":156,"code":["- slow workflow: occurs rarely"]},"column":{"s":2,"e":30}},"dim":["","list.43","listItem.1","paragraph.0","text.0"],"code":"slow workflow: occurs rarely"},{"id":"/root/children/43/children/2","type":"listItem","loc":{"start":3653,"end":3684,"line":{"s":157,"e":157,"code":["- slow workflow takes long time"]},"column":{"s":0,"e":31}},"dim":["","list.43","listItem.2"],"code":"- slow workflow takes long time"},{"id":"/root/children/43/children/2/children/0","type":"paragraph","loc":{"start":3655,"end":3684,"line":{"s":157,"e":157,"code":["- slow workflow takes long time"]},"column":{"s":2,"e":31}},"dim":["","list.43","listItem.2","paragraph.0"],"code":"slow workflow takes long time"},{"id":"/root/children/43/children/2/children/0/children/0","type":"text","loc":{"start":3655,"end":3684,"line":{"s":157,"e":157,"code":["- slow workflow takes long time"]},"column":{"s":2,"e":31}},"dim":["","list.43","listItem.2","paragraph.0","text.0"],"code":"slow workflow takes long time"},{"id":"/root/children/44","type":"paragraph","loc":{"start":3686,"end":3718,"line":{"s":159,"e":159,"code":["You want maximum simplicity, so:"]},"column":{"s":0,"e":32}},"dim":["","paragraph.44"],"code":"You want maximum simplicity, so:"},{"id":"/root/children/44/children/0","type":"text","loc":{"start":3686,"end":3718,"line":{"s":159,"e":159,"code":["You want maximum simplicity, so:"]},"column":{"s":0,"e":32}},"dim":["","paragraph.44","text.0"],"code":"You want maximum simplicity, so:"},{"id":"/root/children/45","type":"paragraph","loc":{"start":3720,"end":3767,"line":{"s":161,"e":161,"code":["✅ Choose Option A: multiple orchestrator queues"]},"column":{"s":0,"e":47}},"dim":["","paragraph.45"],"code":"✅ Choose Option A: multiple orchestrator queues"},{"id":"/root/children/45/children/0","type":"text","loc":{"start":3720,"end":3767,"line":{"s":161,"e":161,"code":["✅ Choose Option A: multiple orchestrator queues"]},"column":{"s":0,"e":47}},"dim":["","paragraph.45","text.0"],"code":"✅ Choose Option A: multiple orchestrator queues"},{"id":"/root/children/46","type":"paragraph","loc":{"start":3769,"end":3787,"line":{"s":163,"e":163,"code":["You only need two:"]},"column":{"s":0,"e":18}},"dim":["","paragraph.46"],"code":"You only need two:"},{"id":"/root/children/46/children/0","type":"text","loc":{"start":3769,"end":3787,"line":{"s":163,"e":163,"code":["You only need two:"]},"column":{"s":0,"e":18}},"dim":["","paragraph.46","text.0"],"code":"You only need two:"},{"id":"/root/children/47","type":"paragraph","loc":{"start":3789,"end":3838,"line":{"s":165,"e":165,"code":["/queue/orchestrator.fast /queue/orchestrator.slow"]},"column":{"s":0,"e":49}},"dim":["","paragraph.47"],"code":"/queue/orchestrator.fast /queue/orchestrator.slow"},{"id":"/root/children/47/children/0","type":"text","loc":{"start":3789,"end":3838,"line":{"s":165,"e":165,"code":["/queue/orchestrator.fast /queue/orchestrator.slow"]},"column":{"s":0,"e":49}},"dim":["","paragraph.47","text.0"],"code":"/queue/orchestrator.fast /queue/orchestrator.slow"},{"id":"/root/children/48","type":"paragraph","loc":{"start":3840,"end":3874,"line":{"s":167,"e":167,"code":["Then route based on workflow type:"]},"column":{"s":0,"e":34}},"dim":["","paragraph.48"],"code":"Then route based on workflow type:"},{"id":"/root/children/48/children/0","type":"text","loc":{"start":3840,"end":3874,"line":{"s":167,"e":167,"code":["Then route based on workflow type:"]},"column":{"s":0,"e":34}},"dim":["","paragraph.48","text.0"],"code":"Then route based on workflow type:"},{"id":"/root/children/49","type":"paragraph","loc":{"start":3876,"end":3978,"line":{"s":169,"e":170,"code":["const destination = workflow.type === 'fast' ?","'/queue/orchestrator.fast' : '/queue/orchestrator.slow'"]},"column":{"s":0,"e":55}},"dim":["","paragraph.49"],"code":"const destination = workflow.type === 'fast' ?\n'/queue/orchestrator.fast' : '/queue/orchestrator.slow'"},{"id":"/root/children/49/children/0","type":"text","loc":{"start":3876,"end":3978,"line":{"s":169,"e":170,"code":["const destination = workflow.type === 'fast' ?","'/queue/orchestrator.fast' : '/queue/orchestrator.slow'"]},"column":{"s":0,"e":55}},"dim":["","paragraph.49","text.0"],"code":"const destination = workflow.type === 'fast' ?\n'/queue/orchestrator.fast' : '/queue/orchestrator.slow'"},{"id":"/root/children/50","type":"paragraph","loc":{"start":3980,"end":4025,"line":{"s":172,"e":172,"code":["Each queue has its own orchestrator consumer."]},"column":{"s":0,"e":45}},"dim":["","paragraph.50"],"code":"Each queue has its own orchestrator consumer."},{"id":"/root/children/50/children/0","type":"text","loc":{"start":3980,"end":4025,"line":{"s":172,"e":172,"code":["Each queue has its own orchestrator consumer."]},"column":{"s":0,"e":45}},"dim":["","paragraph.50","text.0"],"code":"Each queue has its own orchestrator consumer."},{"id":"/root/children/51","type":"paragraph","loc":{"start":4027,"end":4117,"line":{"s":174,"e":175,"code":["✅ Fast workflows never wait behind slow ones ✅ No complexity ✅ No risk ✅","Perfect isolation"]},"column":{"s":0,"e":17}},"dim":["","paragraph.51"],"code":"✅ Fast workflows never wait behind slow ones ✅ No complexity ✅ No risk ✅\nPerfect isolation"},{"id":"/root/children/51/children/0","type":"text","loc":{"start":4027,"end":4117,"line":{"s":174,"e":175,"code":["✅ Fast workflows never wait behind slow ones ✅ No complexity ✅ No risk ✅","Perfect isolation"]},"column":{"s":0,"e":17}},"dim":["","paragraph.51","text.0"],"code":"✅ Fast workflows never wait behind slow ones ✅ No complexity ✅ No risk ✅\nPerfect isolation"},{"id":"/root/children/52","type":"heading","loc":{"start":4119,"end":4131,"line":{"s":177,"e":177,"code":["### sharding"]},"column":{"s":0,"e":12}},"dim":["","heading.52"],"code":"### sharding","symbName":"heading","symbRange":[4133,4343],"symbRangeL":[177,183],"outerCode":"\nMultiple orchestrator queues, each one handles many workflows, but\neach workflow stays in exactly one shard based on\nhash(workflowId)% N. Gives you isolation, parallelism, and ordering with\nbounded queue count.","outerHtml":"\n

Multiple orchestrator queues, each one handles many workflows, but\neach workflow stays in exactly one shard based on\nhash(workflowId)% N. Gives you isolation, parallelism, and ordering with\nbounded queue count.

"},{"id":"/root/children/52/children/0","type":"text","loc":{"start":4123,"end":4131,"line":{"s":177,"e":177,"code":["### sharding"]},"column":{"s":4,"e":12}},"dim":["","heading.52","text.0"],"code":"sharding"},{"id":"/root/children/53","type":"paragraph","loc":{"start":4133,"end":4343,"line":{"s":179,"e":182,"code":["Multiple orchestrator queues, each one handles many workflows, but","each workflow stays in exactly one shard based on","hash(workflowId)% N. Gives you isolation, parallelism, and ordering with","bounded queue count."]},"column":{"s":0,"e":20}},"dim":["","paragraph.53"],"code":"Multiple orchestrator queues, each one handles many workflows, but\neach workflow stays in exactly one shard based on\nhash(workflowId)% N. Gives you isolation, parallelism, and ordering with\nbounded queue count."},{"id":"/root/children/53/children/0","type":"text","loc":{"start":4133,"end":4343,"line":{"s":179,"e":182,"code":["Multiple orchestrator queues, each one handles many workflows, but","each workflow stays in exactly one shard based on","hash(workflowId)% N. Gives you isolation, parallelism, and ordering with","bounded queue count."]},"column":{"s":0,"e":20}},"dim":["","paragraph.53","text.0"],"code":"Multiple orchestrator queues, each one handles many workflows, but\neach workflow stays in exactly one shard based on\nhash(workflowId)% N. Gives you isolation, parallelism, and ordering with\nbounded queue count."},{"id":"/root/children/54","type":"heading","loc":{"start":4345,"end":4358,"line":{"s":184,"e":184,"code":["## conclusion"]},"column":{"s":0,"e":13}},"dim":["","heading.54"],"code":"## conclusion","symbName":"heading","symbRange":[4360,4544],"symbRangeL":[184,190],"outerCode":"\n- we use redis. is redisJSON needed? no.\n- main queue: queue_orchestrator\n- no single long-running program\n- program is executed in steps, steps are a disconnected sequence of messages","outerHtml":"\n
  • we use redis. is redisJSON needed? no.
  • main queue: queue_orchestrator
  • no single long-running program
  • program is executed in steps, steps are a disconnected sequence of messages
"},{"id":"/root/children/54/children/0","type":"text","loc":{"start":4348,"end":4358,"line":{"s":184,"e":184,"code":["## conclusion"]},"column":{"s":3,"e":13}},"dim":["","heading.54","text.0"],"code":"conclusion"},{"id":"/root/children/55","type":"list","loc":{"start":4360,"end":4544,"line":{"s":186,"e":189,"code":["- we use redis. is redisJSON needed? no.","- main queue: queue_orchestrator","- no single long-running program","- program is executed in steps, steps are a disconnected sequence of messages"]},"column":{"s":0,"e":77}},"dim":["","list.55"],"code":"- we use redis. is redisJSON needed? no.\n- main queue: queue_orchestrator\n- no single long-running program\n- program is executed in steps, steps are a disconnected sequence of messages","symbName":"list","symbRange":[4546,4555],"symbRangeL":[186,192],"outerCode":"- main queue: queue_orchestrator\n- no single long-running program\n- program is executed in steps, steps are a disconnected sequence of messages\n\n### state","outerHtml":"
  • main queue: queue_orchestrator
  • no single long-running program
  • program is executed in steps, steps are a disconnected sequence of messages
\n\n

state

"},{"id":"/root/children/55/children/0","type":"listItem","loc":{"start":4360,"end":4400,"line":{"s":186,"e":186,"code":["- we use redis. is redisJSON needed? no."]},"column":{"s":0,"e":40}},"dim":["","list.55","listItem.0"],"code":"- we use redis. is redisJSON needed? no."},{"id":"/root/children/55/children/0/children/0","type":"paragraph","loc":{"start":4362,"end":4400,"line":{"s":186,"e":186,"code":["- we use redis. is redisJSON needed? no."]},"column":{"s":2,"e":40}},"dim":["","list.55","listItem.0","paragraph.0"],"code":"we use redis. is redisJSON needed? no."},{"id":"/root/children/55/children/0/children/0/children/0","type":"text","loc":{"start":4362,"end":4400,"line":{"s":186,"e":186,"code":["- we use redis. is redisJSON needed? no."]},"column":{"s":2,"e":40}},"dim":["","list.55","listItem.0","paragraph.0","text.0"],"code":"we use redis. is redisJSON needed? no."},{"id":"/root/children/55/children/1","type":"listItem","loc":{"start":4401,"end":4433,"line":{"s":187,"e":187,"code":["- main queue: queue_orchestrator"]},"column":{"s":0,"e":32}},"dim":["","list.55","listItem.1"],"code":"- main queue: queue_orchestrator"},{"id":"/root/children/55/children/1/children/0","type":"paragraph","loc":{"start":4403,"end":4433,"line":{"s":187,"e":187,"code":["- main queue: queue_orchestrator"]},"column":{"s":2,"e":32}},"dim":["","list.55","listItem.1","paragraph.0"],"code":"main queue: queue_orchestrator"},{"id":"/root/children/55/children/1/children/0/children/0","type":"text","loc":{"start":4403,"end":4433,"line":{"s":187,"e":187,"code":["- main queue: queue_orchestrator"]},"column":{"s":2,"e":32}},"dim":["","list.55","listItem.1","paragraph.0","text.0"],"code":"main queue: queue_orchestrator"},{"id":"/root/children/55/children/2","type":"listItem","loc":{"start":4434,"end":4466,"line":{"s":188,"e":188,"code":["- no single long-running program"]},"column":{"s":0,"e":32}},"dim":["","list.55","listItem.2"],"code":"- no single long-running program"},{"id":"/root/children/55/children/2/children/0","type":"paragraph","loc":{"start":4436,"end":4466,"line":{"s":188,"e":188,"code":["- no single long-running program"]},"column":{"s":2,"e":32}},"dim":["","list.55","listItem.2","paragraph.0"],"code":"no single long-running program"},{"id":"/root/children/55/children/2/children/0/children/0","type":"text","loc":{"start":4436,"end":4466,"line":{"s":188,"e":188,"code":["- no single long-running program"]},"column":{"s":2,"e":32}},"dim":["","list.55","listItem.2","paragraph.0","text.0"],"code":"no single long-running program"},{"id":"/root/children/55/children/3","type":"listItem","loc":{"start":4467,"end":4544,"line":{"s":189,"e":189,"code":["- program is executed in steps, steps are a disconnected sequence of messages"]},"column":{"s":0,"e":77}},"dim":["","list.55","listItem.3"],"code":"- program is executed in steps, steps are a disconnected sequence of messages"},{"id":"/root/children/55/children/3/children/0","type":"paragraph","loc":{"start":4469,"end":4544,"line":{"s":189,"e":189,"code":["- program is executed in steps, steps are a disconnected sequence of messages"]},"column":{"s":2,"e":77}},"dim":["","list.55","listItem.3","paragraph.0"],"code":"program is executed in steps, steps are a disconnected sequence of messages"},{"id":"/root/children/55/children/3/children/0/children/0","type":"text","loc":{"start":4469,"end":4544,"line":{"s":189,"e":189,"code":["- program is executed in steps, steps are a disconnected sequence of messages"]},"column":{"s":2,"e":77}},"dim":["","list.55","listItem.3","paragraph.0","text.0"],"code":"program is executed in steps, steps are a disconnected sequence of messages"},{"id":"/root/children/56","type":"heading","loc":{"start":4546,"end":4555,"line":{"s":191,"e":191,"code":["### state"]},"column":{"s":0,"e":9}},"dim":["","heading.56"],"code":"### state","symbName":"heading","symbRange":[4557,4903],"symbRangeL":[191,207],"outerCode":"\n- central state of the program\n- who holds the program?\n - one big redis json `program:id`?\n - or flat scalar json namespaced like folders:\n\n```\n/program:id/plan\n/program:id/plan/step:0\n/program:id/outout/step:0\n/program:id/plan/step:1/bag:0\n/program:id/output/step:1/bag:0\n```\n\n- redis prefers small values well granulated into namespaced keys","outerHtml":"\n
  • central state of the program
  • who holds the program?
    • one big redis json `program:id`?
    • or flat scalar json namespaced like folders:
\n\n

```\n/program:id/plan\n/program:id/plan/step:0\n/program:id/outout/step:0\n/program:id/plan/step:1/bag:0\n/program:id/output/step:1/bag:0\n```

\n\n
  • redis prefers small values well granulated into namespaced keys
"},{"id":"/root/children/56/children/0","type":"text","loc":{"start":4550,"end":4555,"line":{"s":191,"e":191,"code":["### state"]},"column":{"s":4,"e":9}},"dim":["","heading.56","text.0"],"code":"state"},{"id":"/root/children/57","type":"list","loc":{"start":4557,"end":4698,"line":{"s":193,"e":196,"code":["- central state of the program","- who holds the program?"," - one big redis json `program:id`?"," - or flat scalar json namespaced like folders:"]},"column":{"s":0,"e":48}},"dim":["","list.57"],"code":"- central state of the program\n- who holds the program?\n - one big redis json `program:id`?\n - or flat scalar json namespaced like folders:","symbName":"list","symbRange":[4700,4836],"symbRangeL":[193,205],"outerCode":"- who holds the program?\n - one big redis json `program:id`?\n - or flat scalar json namespaced like folders:\n\n```\n/program:id/plan\n/program:id/plan/step:0\n/program:id/outout/step:0\n/program:id/plan/step:1/bag:0\n/program:id/output/step:1/bag:0\n```","outerHtml":"
  • who holds the program?
    • one big redis json `program:id`?
    • or flat scalar json namespaced like folders:
\n\n

```\n/program:id/plan\n/program:id/plan/step:0\n/program:id/outout/step:0\n/program:id/plan/step:1/bag:0\n/program:id/output/step:1/bag:0\n```

"},{"id":"/root/children/57/children/0","type":"listItem","loc":{"start":4557,"end":4587,"line":{"s":193,"e":193,"code":["- central state of the program"]},"column":{"s":0,"e":30}},"dim":["","list.57","listItem.0"],"code":"- central state of the program"},{"id":"/root/children/57/children/0/children/0","type":"paragraph","loc":{"start":4559,"end":4587,"line":{"s":193,"e":193,"code":["- central state of the program"]},"column":{"s":2,"e":30}},"dim":["","list.57","listItem.0","paragraph.0"],"code":"central state of the program"},{"id":"/root/children/57/children/0/children/0/children/0","type":"text","loc":{"start":4559,"end":4587,"line":{"s":193,"e":193,"code":["- central state of the program"]},"column":{"s":2,"e":30}},"dim":["","list.57","listItem.0","paragraph.0","text.0"],"code":"central state of the program"},{"id":"/root/children/57/children/1","type":"listItem","loc":{"start":4588,"end":4698,"line":{"s":194,"e":196,"code":["- who holds the program?"," - one big redis json `program:id`?"," - or flat scalar json namespaced like folders:"]},"column":{"s":0,"e":48}},"dim":["","list.57","listItem.1"],"code":"- who holds the program?\n - one big redis json `program:id`?\n - or flat scalar json namespaced like folders:"},{"id":"/root/children/57/children/1/children/0","type":"paragraph","loc":{"start":4590,"end":4612,"line":{"s":194,"e":194,"code":["- who holds the program?"]},"column":{"s":2,"e":24}},"dim":["","list.57","listItem.1","paragraph.0"],"code":"who holds the program?"},{"id":"/root/children/57/children/1/children/0/children/0","type":"text","loc":{"start":4590,"end":4612,"line":{"s":194,"e":194,"code":["- who holds the program?"]},"column":{"s":2,"e":24}},"dim":["","list.57","listItem.1","paragraph.0","text.0"],"code":"who holds the program?"},{"id":"/root/children/57/children/1/children/1","type":"list","loc":{"start":4615,"end":4698,"line":{"s":195,"e":196,"code":[" - one big redis json `program:id`?"," - or flat scalar json namespaced like folders:"]},"column":{"s":2,"e":48}},"dim":["","list.57","listItem.1","list.1"],"code":"- one big redis json `program:id`?\n - or flat scalar json namespaced like folders:","symbName":"list","symbRange":[null,null],"symbRangeL":[195,196],"outerCode":"","outerHtml":""},{"id":"/root/children/57/children/1/children/1/children/0","type":"listItem","loc":{"start":4615,"end":4649,"line":{"s":195,"e":195,"code":[" - one big redis json `program:id`?"]},"column":{"s":2,"e":36}},"dim":["","list.57","listItem.1","list.1","listItem.0"],"code":"- one big redis json `program:id`?"},{"id":"/root/children/57/children/1/children/1/children/0/children/0","type":"paragraph","loc":{"start":4617,"end":4649,"line":{"s":195,"e":195,"code":[" - one big redis json `program:id`?"]},"column":{"s":4,"e":36}},"dim":["","list.57","listItem.1","list.1","listItem.0","paragraph.0"],"code":"one big redis json `program:id`?"},{"id":"/root/children/57/children/1/children/1/children/0/children/0/children/0","type":"text","loc":{"start":4617,"end":4636,"line":{"s":195,"e":195,"code":[" - one big redis json `program:id`?"]},"column":{"s":4,"e":23}},"dim":["","list.57","listItem.1","list.1","listItem.0","paragraph.0","text.0"],"code":"one big redis json "},{"id":"/root/children/57/children/1/children/1/children/0/children/0/children/1","type":"inlineCode","loc":{"start":4636,"end":4648,"line":{"s":195,"e":195,"code":[" - one big redis json `program:id`?"]},"column":{"s":23,"e":35}},"dim":["","list.57","listItem.1","list.1","listItem.0","paragraph.0","inlineCode.1"],"code":"`program:id`"},{"id":"/root/children/57/children/1/children/1/children/0/children/0/children/2","type":"text","loc":{"start":4648,"end":4649,"line":{"s":195,"e":195,"code":[" - one big redis json `program:id`?"]},"column":{"s":35,"e":36}},"dim":["","list.57","listItem.1","list.1","listItem.0","paragraph.0","text.2"],"code":"?"},{"id":"/root/children/57/children/1/children/1/children/1","type":"listItem","loc":{"start":4652,"end":4698,"line":{"s":196,"e":196,"code":[" - or flat scalar json namespaced like folders:"]},"column":{"s":2,"e":48}},"dim":["","list.57","listItem.1","list.1","listItem.1"],"code":"- or flat scalar json namespaced like folders:"},{"id":"/root/children/57/children/1/children/1/children/1/children/0","type":"paragraph","loc":{"start":4654,"end":4698,"line":{"s":196,"e":196,"code":[" - or flat scalar json namespaced like folders:"]},"column":{"s":4,"e":48}},"dim":["","list.57","listItem.1","list.1","listItem.1","paragraph.0"],"code":"or flat scalar json namespaced like folders:"},{"id":"/root/children/57/children/1/children/1/children/1/children/0/children/0","type":"text","loc":{"start":4654,"end":4698,"line":{"s":196,"e":196,"code":[" - or flat scalar json namespaced like folders:"]},"column":{"s":4,"e":48}},"dim":["","list.57","listItem.1","list.1","listItem.1","paragraph.0","text.0"],"code":"or flat scalar json namespaced like folders:"},{"id":"/root/children/58","type":"code","loc":{"start":4700,"end":4836,"line":{"s":198,"e":204,"code":["```","/program:id/plan","/program:id/plan/step:0","/program:id/outout/step:0","/program:id/plan/step:1/bag:0","/program:id/output/step:1/bag:0","```"]},"column":{"s":0,"e":3}},"dim":["","code.58"],"code":"```\n/program:id/plan\n/program:id/plan/step:0\n/program:id/outout/step:0\n/program:id/plan/step:1/bag:0\n/program:id/output/step:1/bag:0\n```","symbName":"code","symbRange":[4838,5069],"symbRangeL":[null,215],"outerCode":";{ engine: dot, rankdir: TD, applyCss: dark, useGvPack: true, clusterize: false, intention: full }\n\n# pattern: orchestrator\n\n- uses 3 queues\n- structure: serial, exclusive+parallel, serial\n- example:\n - `[ serial tasks ]`\n - `[ bag of parallel tasks ]`\n - `[ untiloop ]`\n\n## experimental\n\n- TODO: could also support eval `code` ?\n- holds program state in namespaced redis keys: flat json\n- state can collect output of tasks in adjacent structure\n\n## Recommended minimal clean setup (3 queues)\n\n- /queue/orchestrator\n - 100% serial queue: prefetch = 1, 1 consumer only\n - State machine messages only (continue workflow, step done, parallel done,\n start workflow, etc.)\n\n- /queue/serial-tasks\n - 100% serial queue: prefetch = 1, 1 consumer only\n - tasks run sequentially by nature\n\n- /queue/parallel-tasks\n - many consumers\n - tasks run concurrently\n\n- each task updates its state after finished\n - TODO: but how?\n - completion message\n\nThis is the simplest clean design that gives you:\n\n- controlled sequential execution\n- unlimited parallelism\n- simple, readable routing\n- non-interfering logic\n\n### queues\n\n- queue_orchestrator\n- queue_tasks_serial\n- queue_tasks_parallel\n\n## Workflow Orchestration\n\nRabbitMQ is only the transport; the orchestration logic lives in\nthe program-json.\n\nThis is a “Workflow Orchestration” (a.k.a.\nSaga Orchestration) pattern, a classic Saga orchestration where every “step”\nis either:\n\n- a sequential action, or\n- a parallel fan-out group with a join barrier.\n\nOrchestrated Workflow with Barrier Synchronization\n\n- Reads the workflow definition (your array).\n- For each step:\n- If it’s a single task → publish to a serial queue, wait for ack.\n- If it’s a parallel block → fan-out messages to a parallel queue,\n track N acks, wait until all are done (barrier),continue to next step.\n\n## problem: wait for bag of tasks\n\n- its called Barrier of orchestration\n- or logical cut\n- its how to wait all parallel tasks have finished, and we shall proceed with\n next tasks\n\n### Redis Pub/Sub notification\n\n- Redis: (best).\n\n- Setup\n\nWhen you publish the parallel tasks:\n\n```\nSet counter:\nSET barrier:: N\n\nWorkers, when done:\n\nDECR barrier::\n\nIf result == 0 →\nPUBLISH barrier:done \":\"\n```\n\nOrchestrator waits:\n\n```\nredisSub.subscribe(\"barrier:done\");\nredisSub.on(\"message\", (channel, msg) => {\n const [wf, step] = msg.split(\":\");\n continueWorkflow(wf, step);\n});\n```\n\n✅ zero polling ✅ instant reaction ✅ very scalable\n\nThis is the cleanest.\n\n### join barrier\n\n#### Barrier Counter in Redis (recommended)\n\nSimple, robust, works great with RabbitMQ.\n\nHow it works:\n\n- Before publishing N parallel tasks → SET barrier::pending = N\n- Each worker, after finishing, runs: DECR barrier::pending\n- When the value hits 0, the orchestrator continues to next step.\n\nWhy this is best:\n\n- Atomic\n- No race conditions\n- Survives crashes\n\nVery easy in Node.js + STOMP\n\n#### Barrier via “completion messages” queue\n\nRabbitMQ-only solution. Pattern:\n\n- Each parallel task sends a completion message to a dedicated queue:\n `workflow..step..completed`\n- Orchestrator counts how many \"completed\" messages arrived.\n- When it reaches N → continue.\n\nCons:\n\n- Requires the orchestrator to stay online and track counts in memory or\n storage.\n- Not safe without persistent storage.\n- Use only when you cannot add Redis/Postgres.\n\n## problem: parallel programs?\n\n- can multiple programs run in the same mq-space?\n- will a slow program not hold up a fast one?\n- possible solution: sharding into few queue's\n\nYou said:\n\n- fast workflow: occurs often\n- slow workflow: occurs rarely\n- slow workflow takes long time\n\nYou want maximum simplicity, so:\n\n✅ Choose Option A: multiple orchestrator queues\n\nYou only need two:\n\n/queue/orchestrator.fast /queue/orchestrator.slow\n\nThen route based on workflow type:\n\nconst destination = workflow.type === 'fast' ?\n'/queue/orchestrator.fast' : '/queue/orchestrator.slow'\n\nEach queue has its own orchestrator consumer.\n\n✅ Fast workflows never wait behind slow ones ✅ No complexity ✅ No risk ✅\nPerfect isolation\n\n### sharding\n\nMultiple orchestrator queues, each one handles many workflows, but\neach workflow stays in exactly one shard based on\nhash(workflowId)% N. Gives you isolation, parallelism, and ordering with\nbounded queue count.\n\n## conclusion\n\n- we use redis. is redisJSON needed? no.\n- main queue: queue_orchestrator\n- no single long-running program\n- program is executed in steps, steps are a disconnected sequence of messages\n\n### state\n\n- central state of the program\n- who holds the program?\n - one big redis json `program:id`?\n - or flat scalar json namespaced like folders:\n\n```\n/program:id/plan\n/program:id/plan/step:0\n/program:id/outout/step:0\n/program:id/plan/step:1/bag:0\n/program:id/output/step:1/bag:0\n```\n\n- redis prefers small values well granulated into namespaced keys\n\n### task output\n\n- each task when finished, can set its output to\n `/program:id/plan/step:1/{bag:0}`\n- this is done by the consumer, on ACK\n\n### completion message","outerHtml":"

;{ engine: dot, rankdir: TD, applyCss: dark, useGvPack: true, clusterize: false, intention: full }

\n\n

pattern: orchestrator

\n\n
  • uses 3 queues
  • structure: serial, exclusive+parallel, serial
  • example:
    • `[ serial tasks ]`
    • `[ bag of parallel tasks ]`
    • `[ untiloop ]`
\n\n

experimental

\n\n
  • TODO: could also support eval `code` ?
  • holds program state in namespaced redis keys: flat json
  • state can collect output of tasks in adjacent structure
\n\n

Recommended minimal clean setup (3 queues)

\n\n
  • /queue/orchestrator
    • 100% serial queue: prefetch = 1, 1 consumer only
    • State machine messages only (continue workflow, step done, parallel done,\n start workflow, etc.)
\n\n
  • /queue/serial-tasks
    • 100% serial queue: prefetch = 1, 1 consumer only
    • tasks run sequentially by nature
\n\n
  • /queue/parallel-tasks
    • many consumers
    • tasks run concurrently
\n\n
  • each task updates its state after finished
    • TODO: but how?
    • completion message
\n\n

This is the simplest clean design that gives you:

\n\n
  • controlled sequential execution
  • unlimited parallelism
  • simple, readable routing
  • non-interfering logic
\n\n

queues

\n\n
  • queue_orchestrator
  • queue_tasks_serial
  • queue_tasks_parallel
\n\n

Workflow Orchestration

\n\n

RabbitMQ is only the transport; the orchestration logic lives in\nthe program-json.

\n\n

This is a “Workflow Orchestration” (a.k.a.\nSaga Orchestration) pattern, a classic Saga orchestration where every “step”\nis either:

\n\n
  • a sequential action, or
  • a parallel fan-out group with a join barrier.
\n\n

Orchestrated Workflow with Barrier Synchronization

\n\n
  • Reads the workflow definition (your array).
  • For each step:
  • If it’s a single task → publish to a serial queue, wait for ack.
  • If it’s a parallel block → fan-out messages to a parallel queue,\n track N acks, wait until all are done (barrier),continue to next step.
\n\n

problem: wait for bag of tasks

\n\n
  • its called Barrier of orchestration
  • or logical cut
  • its how to wait all parallel tasks have finished, and we shall proceed with\n next tasks
\n\n

Redis Pub/Sub notification

\n\n
  • Redis: (best).
\n\n
  • Setup
\n\n

When you publish the parallel tasks:

\n\n

```\nSet counter:\nSET barrier:: N

\n\n

Workers, when done:

\n\n

DECR barrier::

\n\n

If result == 0 →\nPUBLISH barrier:done \":\"\n```

\n\n

Orchestrator waits:

\n\n

```\nredisSub.subscribe(\"barrier:done\");\nredisSub.on(\"message\", (channel, msg) => {\n const [wf, step] = msg.split(\":\");\n continueWorkflow(wf, step);\n});\n```

\n\n

✅ zero polling ✅ instant reaction ✅ very scalable

\n\n

This is the cleanest.

\n\n

join barrier

\n\n

Barrier Counter in Redis (recommended)

\n\n

Simple, robust, works great with RabbitMQ.

\n\n

How it works:

\n\n
  • Before publishing N parallel tasks → SET barrier::pending = N
  • Each worker, after finishing, runs: DECR barrier::pending
  • When the value hits 0, the orchestrator continues to next step.
\n\n

Why this is best:

\n\n
  • Atomic
  • No race conditions
  • Survives crashes
\n\n

Very easy in Node.js + STOMP

\n\n

Barrier via “completion messages” queue

\n\n

RabbitMQ-only solution. Pattern:

\n\n
  • Each parallel task sends a completion message to a dedicated queue:\n `workflow..step..completed`
  • Orchestrator counts how many \"completed\" messages arrived.
  • When it reaches N → continue.
\n\n

Cons:

\n\n
  • Requires the orchestrator to stay online and track counts in memory or\n storage.
  • Not safe without persistent storage.
  • Use only when you cannot add Redis/Postgres.
\n\n

problem: parallel programs?

\n\n
  • can multiple programs run in the same mq-space?
  • will a slow program not hold up a fast one?
  • possible solution: sharding into few queue's
\n\n

You said:

\n\n
  • fast workflow: occurs often
  • slow workflow: occurs rarely
  • slow workflow takes long time
\n\n

You want maximum simplicity, so:

\n\n

✅ Choose Option A: multiple orchestrator queues

\n\n

You only need two:

\n\n

/queue/orchestrator.fast /queue/orchestrator.slow

\n\n

Then route based on workflow type:

\n\n

const destination = workflow.type === 'fast' ?\n'/queue/orchestrator.fast' : '/queue/orchestrator.slow'

\n\n

Each queue has its own orchestrator consumer.

\n\n

✅ Fast workflows never wait behind slow ones ✅ No complexity ✅ No risk ✅\nPerfect isolation

\n\n

sharding

\n\n

Multiple orchestrator queues, each one handles many workflows, but\neach workflow stays in exactly one shard based on\nhash(workflowId)% N. Gives you isolation, parallelism, and ordering with\nbounded queue count.

\n\n

conclusion

\n\n
  • we use redis. is redisJSON needed? no.
  • main queue: queue_orchestrator
  • no single long-running program
  • program is executed in steps, steps are a disconnected sequence of messages
\n\n

state

\n\n
  • central state of the program
  • who holds the program?
    • one big redis json `program:id`?
    • or flat scalar json namespaced like folders:
\n\n

```\n/program:id/plan\n/program:id/plan/step:0\n/program:id/outout/step:0\n/program:id/plan/step:1/bag:0\n/program:id/output/step:1/bag:0\n```

\n\n
  • redis prefers small values well granulated into namespaced keys
\n\n

task output

\n\n
  • each task when finished, can set its output to\n `/program:id/plan/step:1/{bag:0}`
  • this is done by the consumer, on ACK
\n\n

completion message

"},{"id":"/root/children/59","type":"list","loc":{"start":4838,"end":4903,"line":{"s":206,"e":206,"code":["- redis prefers small values well granulated into namespaced keys"]},"column":{"s":0,"e":65}},"dim":["","list.59"],"code":"- redis prefers small values well granulated into namespaced keys","symbName":"list","symbRange":[4905,4920],"symbRangeL":[206,209],"outerCode":"\n### task output","outerHtml":"\n

task output

"},{"id":"/root/children/59/children/0","type":"listItem","loc":{"start":4838,"end":4903,"line":{"s":206,"e":206,"code":["- redis prefers small values well granulated into namespaced keys"]},"column":{"s":0,"e":65}},"dim":["","list.59","listItem.0"],"code":"- redis prefers small values well granulated into namespaced keys"},{"id":"/root/children/59/children/0/children/0","type":"paragraph","loc":{"start":4840,"end":4903,"line":{"s":206,"e":206,"code":["- redis prefers small values well granulated into namespaced keys"]},"column":{"s":2,"e":65}},"dim":["","list.59","listItem.0","paragraph.0"],"code":"redis prefers small values well granulated into namespaced keys"},{"id":"/root/children/59/children/0/children/0/children/0","type":"text","loc":{"start":4840,"end":4903,"line":{"s":206,"e":206,"code":["- redis prefers small values well granulated into namespaced keys"]},"column":{"s":2,"e":65}},"dim":["","list.59","listItem.0","paragraph.0","text.0"],"code":"redis prefers small values well granulated into namespaced keys"},{"id":"/root/children/60","type":"heading","loc":{"start":4905,"end":4920,"line":{"s":208,"e":208,"code":["### task output"]},"column":{"s":0,"e":15}},"dim":["","heading.60"],"code":"### task output","symbName":"heading","symbRange":[4922,5045],"symbRangeL":[208,213],"outerCode":"\n- each task when finished, can set its output to\n `/program:id/plan/step:1/{bag:0}`\n- this is done by the consumer, on ACK","outerHtml":"\n
  • each task when finished, can set its output to\n `/program:id/plan/step:1/{bag:0}`
  • this is done by the consumer, on ACK
"},{"id":"/root/children/60/children/0","type":"text","loc":{"start":4909,"end":4920,"line":{"s":208,"e":208,"code":["### task output"]},"column":{"s":4,"e":15}},"dim":["","heading.60","text.0"],"code":"task output"},{"id":"/root/children/61","type":"list","loc":{"start":4922,"end":5045,"line":{"s":210,"e":212,"code":["- each task when finished, can set its output to"," `/program:id/plan/step:1/{bag:0}`","- this is done by the consumer, on ACK"]},"column":{"s":0,"e":38}},"dim":["","list.61"],"code":"- each task when finished, can set its output to\n `/program:id/plan/step:1/{bag:0}`\n- this is done by the consumer, on ACK","symbName":"list","symbRange":[5047,5069],"symbRangeL":[210,215],"outerCode":" `/program:id/plan/step:1/{bag:0}`\n- this is done by the consumer, on ACK\n\n### completion message","outerHtml":"

`/program:id/plan/step:1/{bag:0}`

  • this is done by the consumer, on ACK
\n\n

completion message

"},{"id":"/root/children/61/children/0","type":"listItem","loc":{"start":4922,"end":5006,"line":{"s":210,"e":211,"code":["- each task when finished, can set its output to"," `/program:id/plan/step:1/{bag:0}`"]},"column":{"s":0,"e":35}},"dim":["","list.61","listItem.0"],"code":"- each task when finished, can set its output to\n `/program:id/plan/step:1/{bag:0}`"},{"id":"/root/children/61/children/0/children/0","type":"paragraph","loc":{"start":4924,"end":5006,"line":{"s":210,"e":211,"code":["- each task when finished, can set its output to"," `/program:id/plan/step:1/{bag:0}`"]},"column":{"s":2,"e":35}},"dim":["","list.61","listItem.0","paragraph.0"],"code":"each task when finished, can set its output to\n `/program:id/plan/step:1/{bag:0}`"},{"id":"/root/children/61/children/0/children/0/children/0","type":"text","loc":{"start":4924,"end":4971,"line":{"s":210,"e":211,"code":["- each task when finished, can set its output to"," `/program:id/plan/step:1/{bag:0}`"]},"column":{"s":2,"e":0}},"dim":["","list.61","listItem.0","paragraph.0","text.0"],"code":"each task when finished, can set its output to\n"},{"id":"/root/children/61/children/0/children/0/children/1","type":"inlineCode","loc":{"start":4973,"end":5006,"line":{"s":211,"e":211,"code":[" `/program:id/plan/step:1/{bag:0}`"]},"column":{"s":2,"e":35}},"dim":["","list.61","listItem.0","paragraph.0","inlineCode.1"],"code":"`/program:id/plan/step:1/{bag:0}`"},{"id":"/root/children/61/children/1","type":"listItem","loc":{"start":5007,"end":5045,"line":{"s":212,"e":212,"code":["- this is done by the consumer, on ACK"]},"column":{"s":0,"e":38}},"dim":["","list.61","listItem.1"],"code":"- this is done by the consumer, on ACK"},{"id":"/root/children/61/children/1/children/0","type":"paragraph","loc":{"start":5009,"end":5045,"line":{"s":212,"e":212,"code":["- this is done by the consumer, on ACK"]},"column":{"s":2,"e":38}},"dim":["","list.61","listItem.1","paragraph.0"],"code":"this is done by the consumer, on ACK"},{"id":"/root/children/61/children/1/children/0/children/0","type":"text","loc":{"start":5009,"end":5045,"line":{"s":212,"e":212,"code":["- this is done by the consumer, on ACK"]},"column":{"s":2,"e":38}},"dim":["","list.61","listItem.1","paragraph.0","text.0"],"code":"this is done by the consumer, on ACK"},{"id":"/root/children/62","type":"heading","loc":{"start":5047,"end":5069,"line":{"s":214,"e":214,"code":["### completion message"]},"column":{"s":0,"e":22}},"dim":["","heading.62"],"code":"### completion message","symbName":"heading","symbRange":[5071,null],"symbRangeL":[214,215],"outerCode":"","outerHtml":""},{"id":"/root/children/62/children/0","type":"text","loc":{"start":5051,"end":5069,"line":{"s":214,"e":214,"code":["### completion message"]},"column":{"s":4,"e":22}},"dim":["","heading.62","text.0"],"code":"completion message"},{"id":"/root/children/63","type":"list","loc":{"start":5071,"end":5317,"line":{"s":216,"e":221,"code":["- pure rabbit-mq message `program:next`","- advances queue_orchestrator","- must identify which step of program is saying: move on"," - if a bag of tasks:"," - decr counter with redis eval. fires only when counter===0"," - may postprocess its output"]},"column":{"s":0,"e":32}},"dim":["","list.63"],"code":"- pure rabbit-mq message `program:next`\n- advances queue_orchestrator\n- must identify which step of program is saying: move on\n - if a bag of tasks:\n - decr counter with redis eval. fires only when counter===0\n - may postprocess its output","symbName":"list","symbRange":[null,null],"symbRangeL":[216,221],"outerCode":"- advances queue_orchestrator\n- must identify which step of program is saying: move on\n - if a bag of tasks:\n - decr counter with redis eval. fires only when counter===0","outerHtml":"
  • advances queue_orchestrator
  • must identify which step of program is saying: move on
    • if a bag of tasks:
      • decr counter with redis eval. fires only when counter===0
"},{"id":"/root/children/63/children/0","type":"listItem","loc":{"start":5071,"end":5110,"line":{"s":216,"e":216,"code":["- pure rabbit-mq message `program:next`"]},"column":{"s":0,"e":39}},"dim":["","list.63","listItem.0"],"code":"- pure rabbit-mq message `program:next`"},{"id":"/root/children/63/children/0/children/0","type":"paragraph","loc":{"start":5073,"end":5110,"line":{"s":216,"e":216,"code":["- pure rabbit-mq message `program:next`"]},"column":{"s":2,"e":39}},"dim":["","list.63","listItem.0","paragraph.0"],"code":"pure rabbit-mq message `program:next`"},{"id":"/root/children/63/children/0/children/0/children/0","type":"text","loc":{"start":5073,"end":5096,"line":{"s":216,"e":216,"code":["- pure rabbit-mq message `program:next`"]},"column":{"s":2,"e":25}},"dim":["","list.63","listItem.0","paragraph.0","text.0"],"code":"pure rabbit-mq message "},{"id":"/root/children/63/children/0/children/0/children/1","type":"inlineCode","loc":{"start":5096,"end":5110,"line":{"s":216,"e":216,"code":["- pure rabbit-mq message `program:next`"]},"column":{"s":25,"e":39}},"dim":["","list.63","listItem.0","paragraph.0","inlineCode.1"],"code":"`program:next`"},{"id":"/root/children/63/children/1","type":"listItem","loc":{"start":5111,"end":5140,"line":{"s":217,"e":217,"code":["- advances queue_orchestrator"]},"column":{"s":0,"e":29}},"dim":["","list.63","listItem.1"],"code":"- advances queue_orchestrator"},{"id":"/root/children/63/children/1/children/0","type":"paragraph","loc":{"start":5113,"end":5140,"line":{"s":217,"e":217,"code":["- advances queue_orchestrator"]},"column":{"s":2,"e":29}},"dim":["","list.63","listItem.1","paragraph.0"],"code":"advances queue_orchestrator"},{"id":"/root/children/63/children/1/children/0/children/0","type":"text","loc":{"start":5113,"end":5140,"line":{"s":217,"e":217,"code":["- advances queue_orchestrator"]},"column":{"s":2,"e":29}},"dim":["","list.63","listItem.1","paragraph.0","text.0"],"code":"advances queue_orchestrator"},{"id":"/root/children/63/children/2","type":"listItem","loc":{"start":5141,"end":5317,"line":{"s":218,"e":221,"code":["- must identify which step of program is saying: move on"," - if a bag of tasks:"," - decr counter with redis eval. fires only when counter===0"," - may postprocess its output"]},"column":{"s":0,"e":32}},"dim":["","list.63","listItem.2"],"code":"- must identify which step of program is saying: move on\n - if a bag of tasks:\n - decr counter with redis eval. fires only when counter===0\n - may postprocess its output"},{"id":"/root/children/63/children/2/children/0","type":"paragraph","loc":{"start":5143,"end":5197,"line":{"s":218,"e":218,"code":["- must identify which step of program is saying: move on"]},"column":{"s":2,"e":56}},"dim":["","list.63","listItem.2","paragraph.0"],"code":"must identify which step of program is saying: move on"},{"id":"/root/children/63/children/2/children/0/children/0","type":"text","loc":{"start":5143,"end":5197,"line":{"s":218,"e":218,"code":["- must identify which step of program is saying: move on"]},"column":{"s":2,"e":56}},"dim":["","list.63","listItem.2","paragraph.0","text.0"],"code":"must identify which step of program is saying: move on"},{"id":"/root/children/63/children/2/children/1","type":"list","loc":{"start":5200,"end":5317,"line":{"s":219,"e":221,"code":[" - if a bag of tasks:"," - decr counter with redis eval. fires only when counter===0"," - may postprocess its output"]},"column":{"s":2,"e":32}},"dim":["","list.63","listItem.2","list.1"],"code":"- if a bag of tasks:\n - decr counter with redis eval. fires only when counter===0\n - may postprocess its output","symbName":"list","symbRange":[null,null],"symbRangeL":[219,221],"outerCode":" - decr counter with redis eval. fires only when counter===0","outerHtml":"
- decr counter with redis eval. fires only when counter===0
"},{"id":"/root/children/63/children/2/children/1/children/0","type":"listItem","loc":{"start":5200,"end":5317,"line":{"s":219,"e":221,"code":[" - if a bag of tasks:"," - decr counter with redis eval. fires only when counter===0"," - may postprocess its output"]},"column":{"s":2,"e":32}},"dim":["","list.63","listItem.2","list.1","listItem.0"],"code":"- if a bag of tasks:\n - decr counter with redis eval. fires only when counter===0\n - may postprocess its output"},{"id":"/root/children/63/children/2/children/1/children/0/children/0","type":"paragraph","loc":{"start":5202,"end":5220,"line":{"s":219,"e":219,"code":[" - if a bag of tasks:"]},"column":{"s":4,"e":22}},"dim":["","list.63","listItem.2","list.1","listItem.0","paragraph.0"],"code":"if a bag of tasks:"},{"id":"/root/children/63/children/2/children/1/children/0/children/0/children/0","type":"text","loc":{"start":5202,"end":5220,"line":{"s":219,"e":219,"code":[" - if a bag of tasks:"]},"column":{"s":4,"e":22}},"dim":["","list.63","listItem.2","list.1","listItem.0","paragraph.0","text.0"],"code":"if a bag of tasks:"},{"id":"/root/children/63/children/2/children/1/children/0/children/1","type":"list","loc":{"start":5225,"end":5317,"line":{"s":220,"e":221,"code":[" - decr counter with redis eval. fires only when counter===0"," - may postprocess its output"]},"column":{"s":4,"e":32}},"dim":["","list.63","listItem.2","list.1","listItem.0","list.1"],"code":"- decr counter with redis eval. fires only when counter===0\n - may postprocess its output","symbName":"list","symbRange":[null,null],"symbRangeL":[220,221],"outerCode":"","outerHtml":""},{"id":"/root/children/63/children/2/children/1/children/0/children/1/children/0","type":"listItem","loc":{"start":5225,"end":5284,"line":{"s":220,"e":220,"code":[" - decr counter with redis eval. fires only when counter===0"]},"column":{"s":4,"e":63}},"dim":["","list.63","listItem.2","list.1","listItem.0","list.1","listItem.0"],"code":"- decr counter with redis eval. fires only when counter===0"},{"id":"/root/children/63/children/2/children/1/children/0/children/1/children/0/children/0","type":"paragraph","loc":{"start":5227,"end":5284,"line":{"s":220,"e":220,"code":[" - decr counter with redis eval. fires only when counter===0"]},"column":{"s":6,"e":63}},"dim":["","list.63","listItem.2","list.1","listItem.0","list.1","listItem.0","paragraph.0"],"code":"decr counter with redis eval. fires only when counter===0"},{"id":"/root/children/63/children/2/children/1/children/0/children/1/children/0/children/0/children/0","type":"text","loc":{"start":5227,"end":5284,"line":{"s":220,"e":220,"code":[" - decr counter with redis eval. fires only when counter===0"]},"column":{"s":6,"e":63}},"dim":["","list.63","listItem.2","list.1","listItem.0","list.1","listItem.0","paragraph.0","text.0"],"code":"decr counter with redis eval. fires only when counter===0"},{"id":"/root/children/63/children/2/children/1/children/0/children/1/children/1","type":"listItem","loc":{"start":5289,"end":5317,"line":{"s":221,"e":221,"code":[" - may postprocess its output"]},"column":{"s":4,"e":32}},"dim":["","list.63","listItem.2","list.1","listItem.0","list.1","listItem.1"],"code":"- may postprocess its output"},{"id":"/root/children/63/children/2/children/1/children/0/children/1/children/1/children/0","type":"paragraph","loc":{"start":5291,"end":5317,"line":{"s":221,"e":221,"code":[" - may postprocess its output"]},"column":{"s":6,"e":32}},"dim":["","list.63","listItem.2","list.1","listItem.0","list.1","listItem.1","paragraph.0"],"code":"may postprocess its output"},{"id":"/root/children/63/children/2/children/1/children/0/children/1/children/1/children/0/children/0","type":"text","loc":{"start":5291,"end":5317,"line":{"s":221,"e":221,"code":[" - may postprocess its output"]},"column":{"s":6,"e":32}},"dim":["","list.63","listItem.2","list.1","listItem.0","list.1","listItem.1","paragraph.0","text.0"],"code":"may postprocess its output"}]