Allow timeout in active states
While reviewing the API--Worker split spec we discussed the issues following from selecting oslo.messaging as the queue implementation. Because of the at-most-once semantics of oslo.messaging it may well happen that a task will end up hanging indefinitelly in an active state. This change allows the node_cache.clean_up() function to pick-up these "tasks" and "terminate" them by setting the error state. Change-Id: I696f14d486f10d84aaae5675446138ebdd047ecd Depends-On: Iaeb99ab1954a1d5303c9bd10b81f7f8d6aa7e731
This commit is contained in:
dparalen
parent
de5a3fe03b
commit
ade23652fb
|
|
@ -1,180 +1,230 @@
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Generated by graphviz version 2.36.0 (20140111.2315)
|
||||
<!-- Generated by graphviz version 2.40.1 (20161225.0304)
|
||||
-->
|
||||
<!-- Title: Ironic Inspector states Pages: 1 -->
|
||||
<svg width="838pt" height="340pt"
|
||||
viewBox="0.00 0.00 838.00 340.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 336)">
|
||||
<svg width="823pt" height="318pt"
|
||||
viewBox="0.00 0.00 823.33 317.71" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 313.7101)">
|
||||
<title>Ironic Inspector states</title>
|
||||
<polygon fill="white" stroke="none" points="-4,4 -4,-336 834,-336 834,4 -4,4"/>
|
||||
<polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-313.7101 819.333,-313.7101 819.333,4 -4,4"/>
|
||||
<!-- enrolling -->
|
||||
<g id="node1" class="node"><title>enrolling</title>
|
||||
<ellipse fill="none" stroke="black" cx="34" cy="-223" rx="33.8507" ry="18"/>
|
||||
<text text-anchor="middle" x="34" y="-220.2" font-family="Times,serif" font-size="11.00" fill="gray">enrolling</text>
|
||||
<g id="node1" class="node">
|
||||
<title>enrolling</title>
|
||||
<ellipse fill="none" stroke="#000000" cx="32.7967" cy="-251" rx="32.5946" ry="18"/>
|
||||
<text text-anchor="middle" x="32.7967" y="-247.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">enrolling</text>
|
||||
</g>
|
||||
<!-- processing -->
|
||||
<g id="node2" class="node"><title>processing</title>
|
||||
<ellipse fill="none" stroke="black" cx="175" cy="-276" rx="38.4712" ry="18"/>
|
||||
<text text-anchor="middle" x="175" y="-273.2" font-family="Times,serif" font-size="11.00" fill="gray">processing</text>
|
||||
<g id="node2" class="node">
|
||||
<title>processing</title>
|
||||
<ellipse fill="none" stroke="#000000" cx="168.6961" cy="-193" rx="37.2253" ry="18"/>
|
||||
<text text-anchor="middle" x="168.6961" y="-189.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">processing</text>
|
||||
</g>
|
||||
<!-- enrolling->processing -->
|
||||
<g id="edge1" class="edge"><title>enrolling->processing</title>
|
||||
<path fill="none" stroke="black" d="M61.8006,-233.219C82.5093,-241.115 111.476,-252.16 134.789,-261.049"/>
|
||||
<polygon fill="black" stroke="black" points="133.803,-264.419 144.394,-264.711 136.297,-257.878 133.803,-264.419"/>
|
||||
<text text-anchor="middle" x="102" y="-257" font-family="Times,serif" font-size="10.00">process</text>
|
||||
<g id="edge1" class="edge">
|
||||
<title>enrolling->processing</title>
|
||||
<path fill="none" stroke="#000000" d="M57.4608,-238.975C65.7192,-235.0484 75.0087,-230.7387 83.5934,-227 98.5268,-220.4964 115.1262,-213.7571 129.6709,-208.0106"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="131.336,-211.1171 139.3664,-204.206 128.7789,-204.6009 131.336,-211.1171"/>
|
||||
<text text-anchor="middle" x="98.5885" y="-229" font-family="Times,serif" font-size="10.00" fill="#000000">process</text>
|
||||
</g>
|
||||
<!-- error -->
|
||||
<g id="node3" class="node"><title>error</title>
|
||||
<ellipse fill="none" stroke="black" cx="297" cy="-157" rx="27" ry="18"/>
|
||||
<text text-anchor="middle" x="297" y="-154.2" font-family="Times,serif" font-size="11.00" fill="red">error</text>
|
||||
<g id="node3" class="node">
|
||||
<title>error</title>
|
||||
<ellipse fill="none" stroke="#000000" cx="299.3604" cy="-226" rx="27" ry="18"/>
|
||||
<text text-anchor="middle" x="299.3604" y="-222.7" font-family="Times,serif" font-size="11.00" fill="#ff0000">error</text>
|
||||
</g>
|
||||
<!-- enrolling->error -->
|
||||
<g id="edge2" class="edge"><title>enrolling->error</title>
|
||||
<path fill="none" stroke="black" d="M67.9045,-222.601C111.571,-221.026 190.331,-214.501 252,-189 258.803,-186.187 265.542,-182.121 271.625,-177.825"/>
|
||||
<polygon fill="black" stroke="black" points="274.138,-180.312 280,-171.487 269.914,-174.731 274.138,-180.312"/>
|
||||
<text text-anchor="middle" x="175" y="-220" font-family="Times,serif" font-size="10.00" fill="red">error</text>
|
||||
<g id="edge2" class="edge">
|
||||
<title>enrolling->error</title>
|
||||
<path fill="none" stroke="#000000" d="M61.5155,-260.0788C104.8413,-272.1933 188.8173,-289.4497 254.3604,-265 263.2717,-261.6758 271.5898,-255.704 278.5901,-249.4169"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="281.4462,-251.5201 286.1592,-242.0312 276.5575,-246.51 281.4462,-251.5201"/>
|
||||
<text text-anchor="middle" x="168.6961" y="-279" font-family="Times,serif" font-size="10.00" fill="#ff0000">error</text>
|
||||
</g>
|
||||
<!-- enrolling->error -->
|
||||
<g id="edge3" class="edge">
|
||||
<title>enrolling->error</title>
|
||||
<path fill="none" stroke="#000000" d="M65.2161,-247.9595C114.932,-243.2968 209.7382,-234.4053 262.1742,-229.4876"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="262.7503,-232.949 272.3797,-228.5304 262.0966,-225.9796 262.7503,-232.949"/>
|
||||
<text text-anchor="middle" x="168.6961" y="-243" font-family="Times,serif" font-size="10.00" fill="#ff0000">timeout</text>
|
||||
</g>
|
||||
<!-- processing->error -->
|
||||
<g id="edge11" class="edge"><title>processing->error</title>
|
||||
<path fill="none" stroke="black" d="M192.685,-259.454C213.838,-238.478 250.369,-202.251 273.842,-178.973"/>
|
||||
<polygon fill="black" stroke="black" points="276.333,-181.433 280.969,-171.906 271.404,-176.462 276.333,-181.433"/>
|
||||
<text text-anchor="middle" x="242" y="-223" font-family="Times,serif" font-size="10.00" fill="red">error</text>
|
||||
<g id="edge11" class="edge">
|
||||
<title>processing->error</title>
|
||||
<path fill="none" stroke="#000000" d="M201.6682,-201.3273C220.6372,-206.118 244.4755,-212.1385 263.8552,-217.033"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="263.2653,-220.4938 273.8179,-219.5491 264.9794,-213.7069 263.2653,-220.4938"/>
|
||||
<text text-anchor="middle" x="239.0845" y="-216" font-family="Times,serif" font-size="10.00" fill="#ff0000">error</text>
|
||||
</g>
|
||||
<!-- processing->error -->
|
||||
<g id="edge12" class="edge">
|
||||
<title>processing->error</title>
|
||||
<path fill="none" stroke="#000000" d="M204.4911,-188.0808C220.1547,-187.1698 238.5301,-187.8176 254.3604,-193 261.9563,-195.4867 269.3299,-199.7979 275.8356,-204.4993"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="274.0875,-207.5778 284.1084,-211.0176 278.4198,-202.0795 274.0875,-207.5778"/>
|
||||
<text text-anchor="middle" x="239.0845" y="-195" font-family="Times,serif" font-size="10.00" fill="#ff0000">timeout</text>
|
||||
</g>
|
||||
<!-- finished -->
|
||||
<g id="node6" class="node"><title>finished</title>
|
||||
<ellipse fill="none" stroke="black" cx="568" cy="-176" rx="31.0408" ry="18"/>
|
||||
<text text-anchor="middle" x="568" y="-173.2" font-family="Times,serif" font-size="11.00" fill="gray">finished</text>
|
||||
<g id="node6" class="node">
|
||||
<title>finished</title>
|
||||
<ellipse fill="none" stroke="#000000" cx="563.6084" cy="-212" rx="29.8518" ry="18"/>
|
||||
<text text-anchor="middle" x="563.6084" y="-208.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">finished</text>
|
||||
</g>
|
||||
<!-- processing->finished -->
|
||||
<g id="edge10" class="edge"><title>processing->finished</title>
|
||||
<path fill="none" stroke="black" d="M213.09,-278.731C272.125,-281.22 388.862,-278.362 468,-226 480.338,-217.837 474.066,-206.744 486,-198 497.918,-189.268 513.101,-184.002 527.094,-180.826"/>
|
||||
<polygon fill="black" stroke="black" points="527.879,-184.238 537.009,-178.863 526.519,-177.371 527.879,-184.238"/>
|
||||
<text text-anchor="middle" x="358" y="-274" font-family="Times,serif" font-size="10.00">finish</text>
|
||||
<g id="edge13" class="edge">
|
||||
<title>processing->finished</title>
|
||||
<path fill="none" stroke="#000000" d="M202.9633,-185.8003C209.8363,-184.6342 217.0289,-183.6101 223.8086,-183 237.3325,-181.783 240.7966,-182.3654 254.3604,-183 280.5977,-184.2275 446.0264,-200.3845 523.5778,-208.035"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="523.6182,-211.5559 533.9138,-209.0556 524.3062,-204.5898 523.6182,-211.5559"/>
|
||||
<text text-anchor="middle" x="359.6362" y="-195" font-family="Times,serif" font-size="10.00" fill="#000000">finish</text>
|
||||
</g>
|
||||
<!-- error->error -->
|
||||
<g id="edge5" class="edge"><title>error->error</title>
|
||||
<path fill="none" stroke="black" d="M292.146,-174.782C291.322,-184.315 292.939,-193 297,-193 299.474,-193 301.042,-189.775 301.702,-185.098"/>
|
||||
<polygon fill="black" stroke="black" points="305.206,-184.832 301.854,-174.782 298.207,-184.729 305.206,-184.832"/>
|
||||
<text text-anchor="middle" x="297" y="-196" font-family="Times,serif" font-size="10.00" fill="red">abort</text>
|
||||
<g id="edge5" class="edge">
|
||||
<title>error->error</title>
|
||||
<path fill="none" stroke="#000000" d="M293.9675,-243.7817C293.051,-253.3149 294.8486,-262 299.3604,-262 302.1097,-262 303.8512,-258.7749 304.5848,-254.0981"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="308.0895,-253.8376 304.7533,-243.7817 301.0904,-253.7232 308.0895,-253.8376"/>
|
||||
<text text-anchor="middle" x="299.3604" y="-264" font-family="Times,serif" font-size="10.00" fill="#ff0000">abort</text>
|
||||
</g>
|
||||
<!-- error->error -->
|
||||
<g id="edge6" class="edge"><title>error->error</title>
|
||||
<path fill="none" stroke="black" d="M289.026,-174.42C284.481,-191.791 287.139,-211 297,-211 304.935,-211 308.206,-198.562 306.813,-184.652"/>
|
||||
<polygon fill="black" stroke="black" points="310.188,-183.643 304.974,-174.42 303.298,-184.882 310.188,-183.643"/>
|
||||
<text text-anchor="middle" x="297" y="-214" font-family="Times,serif" font-size="10.00" fill="red">error</text>
|
||||
<g id="edge6" class="edge">
|
||||
<title>error->error</title>
|
||||
<path fill="none" stroke="#000000" d="M290.5823,-243.1418C285.4187,-260.585 288.3447,-280 299.3604,-280 308.3106,-280 311.9204,-267.183 310.1898,-253.0206"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="313.5985,-252.2213 308.1384,-243.1418 306.7447,-253.6445 313.5985,-252.2213"/>
|
||||
<text text-anchor="middle" x="299.3604" y="-282" font-family="Times,serif" font-size="10.00" fill="#ff0000">error</text>
|
||||
</g>
|
||||
<!-- starting -->
|
||||
<g id="node4" class="node"><title>starting</title>
|
||||
<ellipse fill="none" stroke="black" cx="684" cy="-176" rx="30.1339" ry="18"/>
|
||||
<text text-anchor="middle" x="684" y="-173.2" font-family="Times,serif" font-size="11.00" fill="gray">starting</text>
|
||||
<g id="node4" class="node">
|
||||
<title>starting</title>
|
||||
<ellipse fill="none" stroke="#000000" cx="675.5923" cy="-160" rx="28.6835" ry="18"/>
|
||||
<text text-anchor="middle" x="675.5923" y="-156.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">starting</text>
|
||||
</g>
|
||||
<!-- error->starting -->
|
||||
<g id="edge3" class="edge"><title>error->starting</title>
|
||||
<path fill="none" stroke="black" d="M312.309,-171.863C329.454,-188.543 359.908,-214.741 392,-226 479.268,-256.617 511.56,-259.045 600,-232 622.02,-225.266 643.476,-210.507 659.055,-197.83"/>
|
||||
<polygon fill="black" stroke="black" points="661.685,-200.19 667.071,-191.067 657.171,-194.84 661.685,-200.19"/>
|
||||
<text text-anchor="middle" x="502" y="-253" font-family="Times,serif" font-size="10.00">start</text>
|
||||
<g id="edge4" class="edge">
|
||||
<title>error->starting</title>
|
||||
<path fill="none" stroke="#000000" d="M307.1146,-243.415C314.5609,-258.0993 327.2094,-278.5068 344.3604,-290 389.8701,-320.4969 412.8197,-308.1768 467.1308,-301 524.8055,-293.3788 545.1824,-299.3507 593.5342,-267 624.2441,-246.4529 648.2785,-210.5684 662.1952,-186.0695"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="665.2701,-187.7415 667.0264,-177.2933 659.1378,-184.3658 665.2701,-187.7415"/>
|
||||
<text text-anchor="middle" x="500.4067" y="-300" font-family="Times,serif" font-size="10.00" fill="#000000">start</text>
|
||||
</g>
|
||||
<!-- reapplying -->
|
||||
<g id="node5" class="node"><title>reapplying</title>
|
||||
<ellipse fill="none" stroke="black" cx="430" cy="-170" rx="37.7689" ry="18"/>
|
||||
<text text-anchor="middle" x="430" y="-167.2" font-family="Times,serif" font-size="11.00" fill="gray">reapplying</text>
|
||||
<g id="node5" class="node">
|
||||
<title>reapplying</title>
|
||||
<ellipse fill="none" stroke="#000000" cx="430.0215" cy="-246" rx="37.219" ry="18"/>
|
||||
<text text-anchor="middle" x="430.0215" y="-242.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">reapplying</text>
|
||||
</g>
|
||||
<!-- error->reapplying -->
|
||||
<g id="edge4" class="edge"><title>error->reapplying</title>
|
||||
<path fill="none" stroke="black" d="M321.518,-164.6C328.078,-166.394 335.256,-168.057 342,-169 354.867,-170.798 368.908,-171.501 381.882,-171.642"/>
|
||||
<polygon fill="black" stroke="black" points="382.014,-175.142 392.015,-171.644 382.016,-168.142 382.014,-175.142"/>
|
||||
<text text-anchor="middle" x="358" y="-175" font-family="Times,serif" font-size="10.00">reapply</text>
|
||||
<g id="edge7" class="edge">
|
||||
<title>error->reapplying</title>
|
||||
<path fill="none" stroke="#000000" d="M310.0907,-242.8496C318.0668,-253.6986 330.0232,-266.878 344.3604,-273 362.3888,-280.6982 383.3689,-274.3239 400.0536,-265.8422"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="401.7755,-268.8896 408.835,-260.9893 398.3897,-262.7629 401.7755,-268.8896"/>
|
||||
<text text-anchor="middle" x="359.6362" y="-278" font-family="Times,serif" font-size="10.00" fill="#000000">reapply</text>
|
||||
</g>
|
||||
<!-- starting->error -->
|
||||
<g id="edge17" class="edge"><title>starting->error</title>
|
||||
<path fill="none" stroke="black" d="M658.355,-166.246C642.057,-160.232 620.074,-152.939 600,-149 509.248,-131.191 484.23,-136.169 392,-143 372.651,-144.433 351.243,-147.504 333.751,-150.405"/>
|
||||
<polygon fill="black" stroke="black" points="332.686,-147.036 323.42,-152.173 333.867,-153.936 332.686,-147.036"/>
|
||||
<text text-anchor="middle" x="502" y="-141" font-family="Times,serif" font-size="10.00" fill="red">timeout</text>
|
||||
<g id="edge19" class="edge">
|
||||
<title>starting->error</title>
|
||||
<path fill="none" stroke="#000000" d="M647.5816,-164.9137C580.3863,-176.7014 410.0889,-206.5756 335.4011,-219.6776"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="334.7624,-216.2361 325.5176,-221.4114 335.9719,-223.1308 334.7624,-216.2361"/>
|
||||
<text text-anchor="middle" x="500.4067" y="-195" font-family="Times,serif" font-size="10.00" fill="#ff0000">error</text>
|
||||
</g>
|
||||
<!-- starting->error -->
|
||||
<g id="edge18" class="edge"><title>starting->error</title>
|
||||
<path fill="none" stroke="black" d="M662.327,-163.45C646.013,-154.197 622.354,-142.186 600,-136 510.901,-111.341 483.632,-121.737 392,-134 369.759,-136.976 363.775,-135.581 342,-141 338.118,-141.966 334.102,-143.152 330.147,-144.438"/>
|
||||
<polygon fill="black" stroke="black" points="328.852,-141.183 320.554,-147.771 331.149,-147.795 328.852,-141.183"/>
|
||||
<text text-anchor="middle" x="502" y="-125" font-family="Times,serif" font-size="10.00" fill="red">error</text>
|
||||
<g id="edge20" class="edge">
|
||||
<title>starting->error</title>
|
||||
<path fill="none" stroke="#000000" d="M647.1072,-156.3088C587.9227,-149.8806 448.4993,-141.252 344.3604,-186 335.3329,-189.8791 326.8678,-196.3025 319.7692,-202.8788"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="317.2676,-200.4304 312.6408,-209.9616 322.2015,-205.396 317.2676,-200.4304"/>
|
||||
<text text-anchor="middle" x="500.4067" y="-155" font-family="Times,serif" font-size="10.00" fill="#ff0000">timeout</text>
|
||||
</g>
|
||||
<!-- starting->starting -->
|
||||
<g id="edge15" class="edge"><title>starting->starting</title>
|
||||
<path fill="none" stroke="black" d="M674.077,-193.037C672.052,-202.858 675.359,-212 684,-212 689.4,-212 692.718,-208.429 693.952,-203.353"/>
|
||||
<polygon fill="black" stroke="black" points="697.451,-203.028 693.923,-193.037 690.451,-203.047 697.451,-203.028"/>
|
||||
<text text-anchor="middle" x="684" y="-215" font-family="Times,serif" font-size="10.00">start</text>
|
||||
<g id="edge18" class="edge">
|
||||
<title>starting->starting</title>
|
||||
<path fill="none" stroke="#000000" d="M666.0235,-177.0373C664.0706,-186.8579 667.2603,-196 675.5923,-196 680.7998,-196 683.9986,-192.4289 685.1886,-187.3529"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="688.6878,-187.0279 685.1611,-177.0373 681.6878,-187.0466 688.6878,-187.0279"/>
|
||||
<text text-anchor="middle" x="675.5923" y="-198" font-family="Times,serif" font-size="10.00" fill="#000000">start</text>
|
||||
</g>
|
||||
<!-- waiting -->
|
||||
<g id="node7" class="node"><title>waiting</title>
|
||||
<ellipse fill="none" stroke="black" cx="800" cy="-176" rx="30.1339" ry="18"/>
|
||||
<text text-anchor="middle" x="800" y="-173.2" font-family="Times,serif" font-size="11.00" fill="gray">waiting</text>
|
||||
<g id="node7" class="node">
|
||||
<title>waiting</title>
|
||||
<ellipse fill="none" stroke="#000000" cx="786.4917" cy="-94" rx="28.6835" ry="18"/>
|
||||
<text text-anchor="middle" x="786.4917" y="-90.7" font-family="Times,serif" font-size="11.00" fill="#c0c0c0">waiting</text>
|
||||
</g>
|
||||
<!-- starting->waiting -->
|
||||
<g id="edge16" class="edge"><title>starting->waiting</title>
|
||||
<path fill="none" stroke="black" d="M709.356,-185.966C722.001,-189.989 737.749,-193.219 752,-191 755.98,-190.38 760.085,-189.482 764.133,-188.428"/>
|
||||
<polygon fill="black" stroke="black" points="765.356,-191.718 773.978,-185.56 763.398,-184.997 765.356,-191.718"/>
|
||||
<text text-anchor="middle" x="742" y="-195" font-family="Times,serif" font-size="10.00">wait</text>
|
||||
<g id="edge21" class="edge">
|
||||
<title>starting->waiting</title>
|
||||
<path fill="none" stroke="#000000" d="M702.5733,-153.4965C714.456,-149.9501 728.2706,-144.8597 739.6504,-138 748.967,-132.384 757.9815,-124.6634 765.5726,-117.256"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="768.1441,-119.6326 772.6477,-110.0425 763.1466,-114.731 768.1441,-119.6326"/>
|
||||
<text text-anchor="middle" x="731.042" y="-148" font-family="Times,serif" font-size="10.00" fill="#000000">wait</text>
|
||||
</g>
|
||||
<!-- reapplying->error -->
|
||||
<g id="edge14" class="edge"><title>reapplying->error</title>
|
||||
<path fill="none" stroke="black" d="M397.557,-160.564C389.918,-158.671 381.724,-156.968 374,-156 359.888,-154.232 356.221,-155.838 342,-156 339.46,-156.029 336.837,-156.067 334.196,-156.112"/>
|
||||
<polygon fill="black" stroke="black" points="333.986,-152.615 324.057,-156.31 334.123,-159.614 333.986,-152.615"/>
|
||||
<text text-anchor="middle" x="358" y="-159" font-family="Times,serif" font-size="10.00" fill="red">error</text>
|
||||
<g id="edge14" class="edge">
|
||||
<title>reapplying->error</title>
|
||||
<path fill="none" stroke="#000000" d="M392.6776,-245.2627C377.5745,-244.478 360.0376,-242.9464 344.3604,-240 340.6866,-239.3095 336.8909,-238.4196 333.1337,-237.4234"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="333.9675,-234.0212 323.3886,-234.6023 332.0209,-240.7451 333.9675,-234.0212"/>
|
||||
<text text-anchor="middle" x="359.6362" y="-246" font-family="Times,serif" font-size="10.00" fill="#ff0000">error</text>
|
||||
</g>
|
||||
<!-- reapplying->error -->
|
||||
<g id="edge15" class="edge">
|
||||
<title>reapplying->error</title>
|
||||
<path fill="none" stroke="#000000" d="M400.0453,-235.2106C391.9852,-232.7362 383.2021,-230.4184 374.9121,-229 362.6029,-226.894 349.0089,-225.9389 336.7847,-225.5702"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="336.5235,-222.065 326.4616,-225.383 336.3965,-229.0639 336.5235,-222.065"/>
|
||||
<text text-anchor="middle" x="359.6362" y="-231" font-family="Times,serif" font-size="10.00" fill="#ff0000">timeout</text>
|
||||
</g>
|
||||
<!-- reapplying->reapplying -->
|
||||
<g id="edge12" class="edge"><title>reapplying->reapplying</title>
|
||||
<path fill="none" stroke="black" d="M417.242,-187.037C414.638,-196.858 418.891,-206 430,-206 436.943,-206 441.208,-202.429 442.795,-197.353"/>
|
||||
<polygon fill="black" stroke="black" points="446.294,-197.025 442.758,-187.037 439.294,-197.05 446.294,-197.025"/>
|
||||
<text text-anchor="middle" x="430" y="-209" font-family="Times,serif" font-size="10.00">reapply</text>
|
||||
<g id="edge17" class="edge">
|
||||
<title>reapplying->reapplying</title>
|
||||
<path fill="none" stroke="#000000" d="M417.6174,-263.0373C415.086,-272.8579 419.2207,-282 430.0215,-282 436.7719,-282 440.9185,-278.4289 442.4611,-273.3529"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="445.9601,-273.0251 442.4255,-263.0373 438.9601,-273.0494 445.9601,-273.0251"/>
|
||||
<text text-anchor="middle" x="430.0215" y="-284" font-family="Times,serif" font-size="10.00" fill="#000000">reapply</text>
|
||||
</g>
|
||||
<!-- reapplying->finished -->
|
||||
<g id="edge13" class="edge"><title>reapplying->finished</title>
|
||||
<path fill="none" stroke="black" d="M463.32,-161.414C479.743,-158.191 499.985,-155.964 518,-159 522.798,-159.809 527.753,-161.033 532.585,-162.462"/>
|
||||
<polygon fill="black" stroke="black" points="531.65,-165.839 542.243,-165.614 533.822,-159.184 531.65,-165.839"/>
|
||||
<text text-anchor="middle" x="502" y="-162" font-family="Times,serif" font-size="10.00">finish</text>
|
||||
<g id="edge16" class="edge">
|
||||
<title>reapplying->finished</title>
|
||||
<path fill="none" stroke="#000000" d="M450.9627,-231.1072C460.8422,-224.9214 473.0878,-218.4019 485.1308,-215 497.3293,-211.5542 511.0178,-210.2155 523.5053,-209.9079"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="523.6669,-213.4074 533.6532,-209.8684 523.6396,-206.4074 523.6669,-213.4074"/>
|
||||
<text text-anchor="middle" x="500.4067" y="-217" font-family="Times,serif" font-size="10.00" fill="#000000">finish</text>
|
||||
</g>
|
||||
<!-- finished->starting -->
|
||||
<g id="edge7" class="edge"><title>finished->starting</title>
|
||||
<path fill="none" stroke="black" d="M599.55,-176C613.03,-176 629.046,-176 643.426,-176"/>
|
||||
<polygon fill="black" stroke="black" points="643.547,-179.5 653.547,-176 643.547,-172.5 643.547,-179.5"/>
|
||||
<text text-anchor="middle" x="627" y="-179" font-family="Times,serif" font-size="10.00">start</text>
|
||||
<g id="edge8" class="edge">
|
||||
<title>finished->starting</title>
|
||||
<path fill="none" stroke="#000000" d="M587.3573,-200.9721C603.5003,-193.4761 625.0675,-183.4613 642.8382,-175.2095"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="644.6658,-178.2198 652.2616,-170.8337 641.7177,-171.8709 644.6658,-178.2198"/>
|
||||
<text text-anchor="middle" x="620.1426" y="-191" font-family="Times,serif" font-size="10.00" fill="#000000">start</text>
|
||||
</g>
|
||||
<!-- finished->reapplying -->
|
||||
<g id="edge8" class="edge"><title>finished->reapplying</title>
|
||||
<path fill="none" stroke="black" d="M536.608,-175C521.407,-174.468 502.728,-173.766 486,-173 483.316,-172.877 480.559,-172.745 477.777,-172.607"/>
|
||||
<polygon fill="black" stroke="black" points="477.847,-169.106 467.681,-172.09 477.488,-176.097 477.847,-169.106"/>
|
||||
<text text-anchor="middle" x="502" y="-178" font-family="Times,serif" font-size="10.00">reapply</text>
|
||||
<g id="edge10" class="edge">
|
||||
<title>finished->reapplying</title>
|
||||
<path fill="none" stroke="#000000" d="M537.9592,-221.8699C530.8139,-224.4075 523.0032,-226.9866 515.6826,-229 502.7631,-232.5532 488.5551,-235.6791 475.4878,-238.2493"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="474.4681,-234.8805 465.2991,-240.1889 475.7773,-241.757 474.4681,-234.8805"/>
|
||||
<text text-anchor="middle" x="500.4067" y="-238" font-family="Times,serif" font-size="10.00" fill="#000000">reapply</text>
|
||||
</g>
|
||||
<!-- finished->finished -->
|
||||
<g id="edge9" class="edge"><title>finished->finished</title>
|
||||
<path fill="none" stroke="black" d="M557.722,-193.037C555.625,-202.858 559.051,-212 568,-212 573.593,-212 577.029,-208.429 578.307,-203.353"/>
|
||||
<polygon fill="black" stroke="black" points="581.806,-203.027 578.278,-193.037 574.806,-203.047 581.806,-203.027"/>
|
||||
<text text-anchor="middle" x="568" y="-215" font-family="Times,serif" font-size="10.00">finish</text>
|
||||
<g id="edge9" class="edge">
|
||||
<title>finished->finished</title>
|
||||
<path fill="none" stroke="#000000" d="M553.6851,-229.0373C551.66,-238.8579 554.9677,-248 563.6084,-248 569.0087,-248 572.326,-244.4289 573.5601,-239.3529"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="577.0593,-239.0276 573.5316,-229.0373 570.0593,-239.047 577.0593,-239.0276"/>
|
||||
<text text-anchor="middle" x="563.6084" y="-250" font-family="Times,serif" font-size="10.00" fill="#000000">finish</text>
|
||||
</g>
|
||||
<!-- waiting->processing -->
|
||||
<g id="edge19" class="edge"><title>waiting->processing</title>
|
||||
<path fill="none" stroke="black" d="M794.215,-193.78C782.599,-232.499 749.044,-321 685,-321 296,-321 296,-321 296,-321 264.572,-321 231.293,-307.3 207.773,-295.019"/>
|
||||
<polygon fill="black" stroke="black" points="209.41,-291.925 198.95,-290.238 206.075,-298.08 209.41,-291.925"/>
|
||||
<text text-anchor="middle" x="502" y="-324" font-family="Times,serif" font-size="10.00">process</text>
|
||||
<g id="edge23" class="edge">
|
||||
<title>waiting->processing</title>
|
||||
<path fill="none" stroke="#000000" d="M760.7883,-85.4948C711.0858,-69.829 598.0842,-38 500.4067,-38 299.3604,-38 299.3604,-38 299.3604,-38 233.5662,-38 194.187,-120.6512 177.7285,-165.2451"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="174.3425,-164.3214 174.298,-174.9162 180.9398,-166.6616 174.3425,-164.3214"/>
|
||||
<text text-anchor="middle" x="500.4067" y="-40" font-family="Times,serif" font-size="10.00" fill="#000000">process</text>
|
||||
</g>
|
||||
<!-- waiting->error -->
|
||||
<g id="edge21" class="edge"><title>waiting->error</title>
|
||||
<path fill="none" stroke="black" d="M792.906,-158.334C779.466,-123.004 743.73,-47 685,-47 429,-47 429,-47 429,-47 376.609,-47 334.216,-99.2371 312.92,-131.716"/>
|
||||
<polygon fill="black" stroke="black" points="309.881,-129.974 307.482,-140.293 315.793,-133.722 309.881,-129.974"/>
|
||||
<text text-anchor="middle" x="568" y="-50" font-family="Times,serif" font-size="10.00" fill="red">abort</text>
|
||||
<g id="edge24" class="edge">
|
||||
<title>waiting->error</title>
|
||||
<path fill="none" stroke="#000000" d="M757.35,-95.7538C691.4012,-100.2541 526.0942,-114.5295 392.9121,-151 370.4032,-157.1638 363.1085,-157.102 344.3604,-171 333.3652,-179.1507 323.4489,-190.5658 315.7412,-200.8853"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="312.768,-199.0265 309.811,-209.2004 318.4671,-203.0911 312.768,-199.0265"/>
|
||||
<text text-anchor="middle" x="563.6084" y="-124" font-family="Times,serif" font-size="10.00" fill="#ff0000">abort</text>
|
||||
</g>
|
||||
<!-- waiting->error -->
|
||||
<g id="edge22" class="edge"><title>waiting->error</title>
|
||||
<path fill="none" stroke="black" d="M796.115,-157.828C787.838,-112.86 759.631,-0 685,-0 429,-0 429,-0 429,-0 362.169,-0 322.787,-84.9071 306.624,-129.84"/>
|
||||
<polygon fill="black" stroke="black" points="303.315,-128.701 303.354,-139.295 309.931,-130.989 303.315,-128.701"/>
|
||||
<text text-anchor="middle" x="568" y="-3" font-family="Times,serif" font-size="10.00" fill="red">timeout</text>
|
||||
<g id="edge25" class="edge">
|
||||
<title>waiting->error</title>
|
||||
<path fill="none" stroke="#000000" d="M776.681,-76.759C759.8083,-49.6219 722.5372,0 675.5923,0 430.0215,0 430.0215,0 430.0215,0 339.5956,0 310.7541,-137.2788 302.4294,-197.8526"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="298.943,-197.5236 301.1429,-207.8875 305.8862,-198.4138 298.943,-197.5236"/>
|
||||
<text text-anchor="middle" x="563.6084" y="-2" font-family="Times,serif" font-size="10.00" fill="#ff0000">timeout</text>
|
||||
</g>
|
||||
<!-- waiting->starting -->
|
||||
<g id="edge20" class="edge"><title>waiting->starting</title>
|
||||
<path fill="none" stroke="black" d="M769.742,-176C756.059,-176 739.556,-176 724.763,-176"/>
|
||||
<polygon fill="black" stroke="black" points="724.358,-172.5 714.358,-176 724.358,-179.5 724.358,-172.5"/>
|
||||
<text text-anchor="middle" x="742" y="-179" font-family="Times,serif" font-size="10.00">start</text>
|
||||
<g id="edge22" class="edge">
|
||||
<title>waiting->starting</title>
|
||||
<path fill="none" stroke="#000000" d="M763.2435,-105.0099C750.896,-111.097 735.5941,-119.0265 722.4336,-127 715.8825,-130.9691 709.0346,-135.536 702.6611,-139.9821"/>
|
||||
<polygon fill="#000000" stroke="#000000" points="700.4572,-137.2545 694.3364,-145.9024 704.5142,-142.959 700.4572,-137.2545"/>
|
||||
<text text-anchor="middle" x="731.042" y="-129" font-family="Times,serif" font-size="10.00" fill="#000000">start</text>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
|
|
|||
|
Before Width: | Height: | Size: 13 KiB After Width: | Height: | Size: 15 KiB |
|
|
@ -87,6 +87,7 @@ State_space = [
|
|||
'next_states': {
|
||||
Events.error: States.error,
|
||||
Events.process: States.processing,
|
||||
Events.timeout: States.error,
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -111,6 +112,7 @@ State_space = [
|
|||
'next_states': {
|
||||
Events.error: States.error,
|
||||
Events.finish: States.finished,
|
||||
Events.timeout: States.error,
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
@ -119,6 +121,7 @@ State_space = [
|
|||
Events.error: States.error,
|
||||
Events.finish: States.finished,
|
||||
Events.reapply: States.reapplying,
|
||||
Events.timeout: States.error,
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -378,23 +378,25 @@ class TestNodeCacheCleanUp(test_base.NodeTest):
|
|||
|
||||
@mock.patch.object(node_cache, '_get_lock', autospec=True)
|
||||
@mock.patch.object(timeutils, 'utcnow')
|
||||
def test_timeout_starting(self, time_mock, get_lock_mock):
|
||||
def test_timeout_active_state(self, time_mock, get_lock_mock):
|
||||
time_mock.return_value = self.started_at
|
||||
session = db.get_session()
|
||||
db.model_query(db.Node, session=session).filter_by(
|
||||
uuid=self.uuid).update({'state': istate.States.starting})
|
||||
|
||||
CONF.set_override('timeout', 1)
|
||||
current_time = self.started_at + datetime.timedelta(seconds=2)
|
||||
time_mock.return_value = current_time
|
||||
for state in [istate.States.starting, istate.States.enrolling,
|
||||
istate.States.processing, istate.States.reapplying]:
|
||||
db.model_query(db.Node, session=session).filter_by(
|
||||
uuid=self.uuid).update({'state': state, 'finished_at': None})
|
||||
|
||||
self.assertEqual([self.uuid], node_cache.clean_up())
|
||||
current_time = self.started_at + datetime.timedelta(seconds=2)
|
||||
time_mock.return_value = current_time
|
||||
|
||||
res = [(row.state, row.finished_at, row.error) for row in
|
||||
db.model_query(db.Node).all()]
|
||||
self.assertEqual(
|
||||
[(istate.States.error, current_time, 'Introspection timeout')],
|
||||
res)
|
||||
self.assertEqual([self.uuid], node_cache.clean_up())
|
||||
|
||||
res = [(row.state, row.finished_at, row.error) for row in
|
||||
db.model_query(db.Node).all()]
|
||||
self.assertEqual(
|
||||
[(istate.States.error, current_time, 'Introspection timeout')],
|
||||
res)
|
||||
|
||||
def test_old_status(self):
|
||||
CONF.set_override('node_status_keep_time', 42)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
---
|
||||
other:
|
||||
- |
|
||||
Allow a timeout to happen while a node inspection is in any active state.
|
||||
Loading…
Reference in New Issue