Diego Alonso Roque Montoya
03/30/2021, 6:26 PMZach Angell
Diego Alonso Roque Montoya
03/30/2021, 9:45 PMZach Angell
Diego Alonso Roque Montoya
03/30/2021, 10:06 PM*** prefect.utilities.exceptions.ClientError: [{'message': '[{\'extensions\': {\'internal\': {\'statement\': \'WITH "edge__mutation_result_alias" AS (INSERT INTO "public"."edge" ( "mapped", "created", "flow_id", "tenant_id", "upstream_task_id", "key", "id", "updated", "downstream_task_id" ) VALUES (($2)::boolean, DEFAULT, ($1)::uuid, ($3)::uuid, ($4)::uuid, ($5)::text, DEFAULT, DEFAULT, ($6)::uuid), (($8)::boolean, DEFAULT, ($7)::uuid, ($9)::uuid, ($10)::uuid, ($11)::text, DEFAULT, DEFAULT, ($12)::uuid), (($14)::boolean, DEFAULT, ($13)::uuid, ($15)::uuid, ($16)::uuid, ($17)::text, DEFAULT, DEFAULT, ($18)::uuid), (($20)::boolean, DEFAULT, ($19)::uuid, ($21)::uuid, ($22)::uuid, ($23)::text, DEFAULT, DEFAULT, ($24)::uuid), (($26)::boolean, DEFAULT, ($25)::uuid, ($27)::uuid, ($28)::uuid, ($29)::text, DEFAULT, DEFAULT, ($30)::uuid), (($32)::boolean, DEFAULT, ($31)::uuid, ($33)::uuid, ($34)::uuid, ($35)::text, DEFAULT, DEFAULT, ($36)::uuid), (($38)::boolean, DEFAULT, ($37)::uuid, ($39)::uuid, ($40)::uuid, ($41)::text, DEFAULT, DEFAULT, ($42)::uuid), (($44)::boolean, DEFAULT, ($43)::uuid, ($45)::uuid, ($46)::uuid, ($47)::text, DEFAULT, DEFAULT, ($48)::uuid), (($50)::boolean, DEFAULT, ($49)::uuid, ($51)::uuid, ($52)::uuid, ($53)::text, DEFAULT, DEFAULT, ($54)::uuid), (($56)::boolean, DEFAULT, ($55)::uuid, ($57)::uuid, ($58)::uuid, ($59)::text, DEFAULT, DEFAULT, ($60)::uuid), (($62)::boolean, DEFAULT, ($61)::uuid, ($63)::uuid, ($64)::uuid, ($65)::text, DEFAULT, DEFAULT, ($66)::uuid), (($68)::boolean, DEFAULT, ($67)::uuid, ($69)::uuid, ($70)::uuid, ($71)::text, DEFAULT, DEFAULT, ($72)::uuid), (($74)::boolean, DEFAULT, ($73)::uuid, ($75)::uuid, ($76)::uuid, ($77)::text, DEFAULT, DEFAULT, ($78)::uuid), (($80)::boolean, DEFAULT, ($79)::uuid, ($81)::uuid, ($82)::uuid, ($83)::text, DEFAULT, DEFAULT, ($84)::uuid), (($86)::boolean, DEFAULT, ($85)::uuid, ($87)::uuid, ($88)::uuid, ($89)::text, DEFAULT, DEFAULT, ($90)::uuid), (($92)::boolean, DEFAULT, ($91)::uuid, ($93)::uuid, ($94)::uuid, ($95)::text, DEFAULT, DEFAULT, ($96)::uuid), (($98)::boolean, DEFAULT, ($97)::uuid, ($99)::uuid, ($100)::uuid, ($101)::text, DEFAULT, DEFAULT, ($102)::uuid), (($104)::boolean, DEFAULT, ($103)::uuid, ($105)::uuid, ($106)::uuid, ($107)::text, DEFAULT, DEFAULT, ($108)::uuid), (($110)::boolean, DEFAULT, ($109)::uuid, ($111)::uuid, ($112)::uuid, ($113)::text, DEFAULT, DEFAULT, ($114)::uuid), (($116)::boolean, DEFAULT, ($115)::uuid, ($117)::uuid, ($118)::uuid, ($119)::text, DEFAULT, DEFAULT, ($120)::uuid), (($122)::boolean, DEFAULT, ($121)::uuid, ($123)::uuid, ($124)::uuid, ($125)::text, DEFAULT, DEFAULT, ($126)::uuid), (($128)::boolean, DEFAULT, ($127)::uuid, ($129)::uuid, ($130)::uuid, ($131)::text, DEFAULT, DEFAULT, ($132)::uuid), (($134)::boolean, DEFAULT, ($133)::uuid, ($135)::uuid, ($136)::uuid, ($137)::text, DEFAULT, DEFAULT, ($138)::uuid), (($140)::boolean, DEFAULT, ($139)::uuid, ($141)::uuid, ($142)::uuid, ($143)::text, DEFAULT, DEFAULT, ($144)::uuid), (($146)::boolean, DEFAULT, ($145)::uuid, ($147)::uuid, ($148)::uuid, ($149)::text, DEFAULT, DEFAULT, ($150)::uuid), (($152)::boolean, DEFAULT, ($151)::uuid, ($153)::uuid, ($154)::uuid, ($155)::text, DEFAULT, DEFAULT, ($156)::uuid), (($158)::boolean, DEFAULT, ($157)::uuid, ($159)::uuid, ($160)::uuid, ($161)::text, DEFAULT, DEFAULT, ($162)::uuid), (($164)::boolean, DEFAULT, ($163)::uuid, ($165)::uuid, ($166)::uuid, ($167)::text, DEFAULT, DEFAULT, ($168)::uuid), (($170)::boolean, DEFAULT, ($169)::uuid, ($171)::uuid, ($172)::uuid, ($173)::text, DEFAULT, DEFAULT, ($174)::uuid), (($176)::boolean, DEFAULT, ($175)::uuid, ($177)::uuid, ($178)::uuid, ($179)::text, DEFAULT, DEFAULT, ($180)::uuid), (($182)::boolean, DEFAULT, ($181)::uuid, ($183)::uuid, ($184)::uuid, ($185)::text, DEFAULT, DEFAULT, ($186)::uuid), (($188)::boolean, DEFAULT, ($187)::uuid, ($189)::uuid, ($190)::uuid, ($191)::text, DEFAULT, DEFAULT, ($192)::uuid), (($194)::boolean, DEFAULT, ($193)::uuid, ($195)::uuid, ($196)::uuid, ($197)::text, DEFAULT, DEFAULT, ($198)::uuid), (($200)::boolean, DEFAULT, ($199)::uuid, ($201)::uuid, ($202)::uuid, ($203)::text, DEFAULT, DEFAULT, ($204)::uuid), (($206)::boolean, DEFAULT, ($205)::uuid, ($207)::uuid, ($208)::uuid, ($209)::text, DEFAULT, DEFAULT, ($210)::uuid), (($212)::boolean, DEFAULT, ($211)::uuid, ($213)::uuid, ($214)::uuid, ($215)::text, DEFAULT, DEFAULT, ($216)::uuid), (($218)::boolean, DEFAULT, ($217)::uuid, ($219)::uuid, ($220)::uuid, ($221)::text, DEFAULT, DEFAULT, ($222)::uuid), (($224)::boolean, DEFAULT, ($223)::uuid, ($225)::uuid, ($226)::uuid, ($227)::text, DEFAULT, DEFAULT, ($228)::uuid), (($230)::boolean, DEFAULT, ($229)::uuid, ($231)::uuid, ($232)::uuid, ($233)::text, DEFAULT, DEFAULT, ($234)::uuid), (($236)::boolean, DEFAULT, ($235)::uuid, ($237)::uuid, ($238)::uuid, ($239)::text, DEFAULT, DEFAULT, ($240)::uuid), (($242)::boolean, DEFAULT, ($241)::uuid, ($243)::uuid, ($244)::uuid, ($245)::text, DEFAULT, DEFAULT, ($246)::uuid), (($248)::boolean, DEFAULT, ($247)::uuid, ($249)::uuid, ($250)::uuid, ($251)::text, DEFAULT, DEFAULT, ($252)::uuid), (($254)::boolean, DEFAULT, ($253)::uuid, ($255)::uuid, ($256)::uuid, ($257)::text, DEFAULT, DEFAULT, ($258)::uuid), (($260)::boolean, DEFAULT, ($259)::uuid, ($261)::uuid, ($262)::uuid, ($263)::text, DEFAULT, DEFAULT, ($264)::uuid), (($266)::boolean, DEFAULT, ($265)::uuid, ($267)::uuid, ($268)::uuid, ($269)::text, DEFAULT, DEFAULT, ($270)::uuid), (($272)::boolean, DEFAULT, ($271)::uuid, ($273)::uuid, ($274)::uuid, ($275)::text, DEFAULT, DEFAULT, ($276)::uuid), (($278)::boolean, DEFAULT, ($277)::uuid, ($279)::uuid, ($280)::uuid, ($281)::text, DEFAULT, DEFAULT, ($282)::uuid), (($284)::boolean, DEFAULT, ($283)::uuid, ($285)::uuid, ($286)::uuid, ($287)::text, DEFAULT, DEFAULT, ($288)::uuid), (($290)::boolean, DEFAULT, ($289)::uuid, ($291)::uuid, ($292)::uuid, ($293)::text, DEFAULT, DEFAULT, ($294)::uuid), (($296)::boolean, DEFAULT, ($295)::uuid, ($297)::uuid, ($298)::uuid, ($299)::text, DEFAULT, DEFAULT, ($300)::uuid), (($302)::boolean, DEFAULT, ($301)::uuid, ($303)::uuid, ($304)::uuid, ($305)::text, DEFAULT, DEFAULT, ($306)::uuid), (($308)::boolean, DEFAULT, ($307)::uuid, ($309)::uuid, ($310)::uuid, ($311)::text, DEFAULT, DEFAULT, ($312)::uuid), (($314)::boolean, DEFAULT, ($313)::uuid, ($315)::uuid, ($316)::uuid, ($317)::text, DEFAULT, DEFAULT, ($318)::uuid), (($320)::boolean, DEFAULT, ($31
Zach Angell
Diego Alonso Roque Montoya
03/30/2021, 11:21 PMfrom prefect import task, Flow, Task
class MyTask(Task):
def run(self):
return "hello"
tasks = [MyTask() for i in range(3000)]
flow = Flow(name = 'testflow', tasks = tasks)
for i in range(3000):
flow.set_dependencies(task = tasks[i], upstream_tasks= tasks[max(0, i - 10):i] )
print('registering')
flow.register(build=True, project_name='TestProject')
'(Oid 2950,Just ("\\\\170`Lv\\\\166\\\\133O\\\\159\\\\156y\\\\243Zi\\\\196A\\\\148",Binary))\', \'(Oid 2950,Just ("4\\\\226\\\\254\\\\204\\\\249\\\\DC2GZ\\\\168\\\\148\\\\EMx\\\\"\\\\"\\\\223\\\\"",Binary))\', \'(Oid 25,Nothing)\', \'(Oid 2950,Just ("\\\\130na^\\\\173_H\\\\227\\\\129\\\\135;t\\\\CAN\\\\239\\\\166G",Binary))\', \'(Oid 2950,Just ("RJ\\\'P\\\\236\\\\190O\\\\141\\\\166\\\\237\\\\252\\\\SI6rr>",Binary))\', \'(Oid 16,Just ("\\\\NUL",Binary))\', \'(Oid 2950,Just ("\\\\170`Lv\\\\166\\\\133O\\\\159\\\\156y\\\\243Zi\\\\196A\\\\148",Binary))\', \'(Oid 2950,Just ("\\\\130na^\\\\173_H\\\\227\\\\129\\\\135;t\\\\CAN\\\\239\\\\166G",Binary))\', \'(Oid 25,Nothing)\', \'(Oid 2950,Just ("\\\\173\\\\225FEp\\\\240Nd\\\\137\\\\179%\\\\203^\\\\b\\\\187P",Binary))\', \'(Oid 2950,Just ("RJ\\\'P\\\\236\\\\190O\\\\141\\\\166\\\\237\\\\252\\\\SI6rr>",Binary))\', \'(Oid 16,Just ("\\\\NUL",Binary))\', \'(Oid 2950,Just ("\\\\170`Lv\\\\166\\\\133O\\\\159\\\\156y\\\\243Zi\\\\196A\\\\148",Binary))\', \'(Oid 2950,Just ("\\\\130na^\\\\173_H\\\\227\\\\129\\\\135;t\\\\CAN\\\\239\\\\166G",Binary))\', \'(Oid 25,Nothing)\', \'(Oid 2950,Just ("7Kn\\\\181\\\\168]KQ\\\\138\\\\184\\\\241\\\\152=\\\\213\\\\EOT\\\\157",Binary))\', \'(Oid 2950,Just ("RJ\\\'P\\\\236\\\\190O\\\\141\\\\166\\\\237\\\\252\\\\SI6rr>",Binary))\', \'(Oid 16,Just ("\\\\NUL",Binary))\', \'(Oid 2950,Just ("\\\\170`Lv\\\\166\\\\133O\\\\159\\\\156y\\\\243Zi\\\\196A\\\\148",Binary))\', \'(Oid 2950,Just ("\\\\130na^\\\\173_H\\\\227\\\\129\\\\135;t\\\\CAN\\\\239\\\\166G",Binary))\', \'(Oid 25,Nothing)\', \'(Oid 2950,Just ("E\\\\236\\\\137\\\\US\\\\179\\\\151D\\\\142\\\\153\\\\SUB\\\\229\\\\DC2J\\\\211\\\\\\\\C",Binary))\', \'(Oid 2950,Just ("RJ\\\'P\\\\236\\\\190O\\\\141\\\\166\\\\237\\\\252\\\\SI6rr>",Binary))\', \'(Oid 16,Just ("\\\\NUL",Binary))\', \'(Oid 2950,Just ("\\\\170`Lv\\\\166\\\\133O\\\\159\\\\156y\\\\243Zi\\\\196A\\\\148",Binary))\', \'(Oid 2950,Just ("\\\\130na^\\\\173_H\\\\227\\\\129\\\\135;t\\\\CAN\\\\239\\\\166G",Binary))\', \'(Oid 25,Nothing)\', \'(Oid 2950,Just ("\\\\DEL\\\\r\\\\247\\\\243\\\\198rE\\\\f\\\\140R\\\\133\\\\&3O\\\\233\\\\172}",Binary))\', \'(Oid 2950,Just ("RJ\\\'P\\\\236\\\\190O\\\\141\\\\166\\\\237\\\\252\\\\SI6rr>",Binary))\', \'(Oid 16,Just ("\\\\NUL",Binary))\', \'(Oid 2950,Just ("\\\\170`Lv\\\\166\\\\133O\\\\159\\\\156y\\\\243Zi\\\\196A\\\\148",Binary))\', \'(Oid 2950,Just ("\\\\130na^\\\\173_H\\\\227\\\\129\\\\135;t\\\\CAN\\\\239\\\\166G",Binary))\', \'(Oid 25,Nothing)\', \'(Oid 2950,Just ("\\\\ETX\\\\183\\\\211\\\\243?\\\\185F\\\\189\\\\174\\\\155p\\\\144\\\\206\\\\DC2\\\\183\\\\v",Binary))\', \'(Oid 2950,Just ("RJ\\\'P\\\\236\\\\190O\\\\141\\\\166\\\\237\\\\252\\\\SI6rr>",Binary))\', \'(Oid 16,Just ("\\\\NUL",Binary))\', \'(Oid 2950,Just ("\\\\170`Lv\\\\166\\\\133O\\\\159\\\\156y\\\\243Zi\\\\196A\\\\148",Binary))\', \'(Oid 2950,Just ("\\\\130na^\\\\173_H\\\\227\\\\129\\\\135;t\\\\CAN\\\\239\\\\166G",Binary))\', \'(Oid 25,Nothing)\', \'(Oid 2950,Just ("k2M\\\\211\\\\&9}J4\\\\176\\\\216\\\\&4\\\\DLE\\\\244\\\\192aP",Binary))\', \'(Oid 2950,Just ("RJ\\\'P\\\\236\\\\190O\\\\141\\\\166\\\\237\\\\252\\\\SI6rr>",Binary))\', \'(Oid 16,Just ("\\\\NUL",Binary))\', \'(Oid 2950,Just ("\\\\170`Lv\\\\166\\\\133O\\\\159\\\\156y\\\\243Zi\\\\196A\\\\148",Binary))\', \'(Oid 2950,Just ("\\\\130na^\\\\173_H\\\\227\\\\129\\\\135;t\\\\CAN\\\\239\\\\166G",Binary))\', \'(Oid 25,Nothing)\', \'(Oid 2950,Just ("w\\\\CAN1\\\\229\\\\164\\\\&8C4\\\\185\\\\157$\\\\206\\\\200\\\\210-\\\\205",Binary))\', \'(Oid 2950,Just ("RJ\\\'P\\\\236\\\\190O\\\\141\\\\166\\\\237\\\\252\\\\SI6rr>",Binary))\', \'(Oid 16,Just ("\\\\NUL",Binary))\', \'(Oid 2950,Just ("\\\\170`Lv\\\\166\\\\133O\\\\159\\\\156y\\\\243Zi\\\\196A\\\\148",Binary))\', \'(Oid 2950,Just ("\\\\130na^\\\\173_H\\\\227\\\\129\\\\135;t\\\\CAN\\\\239\\\\166G",Binary))\', \'(Oid 25,Nothing)\', \'(Oid 2950,Just ("f\\\\DLE\\\\132\\\\174\\\\150\\\\160B@\\\\154Q\\\\143\\\\DC3qp.\\\\244",Binary))\', \'(Oid 2950,Just ("RJ\\\'P\\\\236\\\\190O\\\\141\\\\166\\\\237\\\\252\\\\SI6rr>",Binary))\', \'(Oid 16,Just ("\\\\NUL",Binary))\', \'(Oid 2950,Just ("\\\\170`Lv\\\\166\\\\133O\\\\159\\\\156y\\\\243Zi\\\\196A\\\\148",Binary))\', \'(Oid 2950,Just ("\\\\130na^\\\\173_H\\\\227\\\\129\\\\135;t\\\\CAN\\\\239\\\\166G",Binary))\', \'(Oid 25,Nothing)\', \'(Oid 2950,Just ("y\\\\189[Mf\\\\128F\\\\ESC\\\\141iZ\\\\CAN\\\\n\\\\142R2",Binary))\', \'(Oid 2950,Just ("RJ\\\'P\\\\236\\\\190O\\\\141\\\\166\\\\237\\\\252\\\\SI6rr>",Binary))\', \'(Oid 16,Just ("\\\\NUL",Binary))\', \'(Oid 2950,Just ("\\\\170`Lv\\\\166\\\\133O\\\\159\\\\156y\\\\243Zi\\\\196A\\\\148",Binary))\', \'(Oid 2950,Just ("\\\\130na^\\\\173_H\\\\227\\\\129\\\\135;t\\\\CAN\\\\239\\\\166G",Binary))\', \'(Oid 25,Nothing)\', \'(Oid 2950,Just ("\\\\198\\\\175\\\\&8\\\\223\\\\171\\\\199B\\\\132\\\\182\\\\169\\\\&1\\\\136\\\\212A?\\\\136",Binary))\']}, \'path\': \'$.selectionSet.insert_flow.args.objects[0].edges.data\', \'code\': \'unexpected\'}, \'message\': \'database query error\'}]'}}}]
Zach Angell
Diego Alonso Roque Montoya
03/31/2021, 12:31 AMZach Angell
Diego Alonso Roque Montoya
04/01/2021, 7:15 PMZach Angell
Diego Alonso Roque Montoya
04/02/2021, 9:29 PMZach Angell
Is it possible to build flow incrementally?Not exactly, but there might be some ways to get around it. The first thing that comes to mind is mapping https://docs.prefect.io/core/concepts/mapping.html. Mapping creates tasks dynamically at run time instead of creating them all on registration.
Diego Alonso Roque Montoya
04/03/2021, 12:35 AMZach Angell
Diego Alonso Roque Montoya
04/05/2021, 6:17 PMZach Angell