Question: inkscape:label="Layer 1" inkscape:groupmode="layer" id="layer1"> <del> <path sodipodi:type="star" style="fill:#2a55f4;fill-opacity:1;stroke:#2a55f4;stroke-width:69;stroke-linejoin:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" id="path3054" sodipodi:sides="7" sodipodi:cx="-588.57141" sodipodi:cy="9.5050402" sodipodi:r1="230.79279" sodipodi:r2="207.9371" sodipodi:arg1="0.54678884" sodipodi:arg2="0.99558779" inkscape:flatsided="true" inkscape:rounded="0" inkscape:randomized="0" d="m -391.42855,129.50504 -168.04608,108.95128 -189.95655,-63.45371 -68.82586,-188.076755 104.1321,-171.074165 198.67647,-25.24924 143.61341,139.588871 z" inkscape:transform-center-x="85.453542" inkscape:transform-center-y="-88.166574" transform="matrix(-0.70423675,-1.1472157,1.1566815,-0.69847357,-57.760066,-127.50052)" inkscape:export-xdpi="90" inkscape:export-ydpi="90" /> </del> <g <del> id="g6172" transform="translate(-0.97949518,-0.04249773)"> </del> <ins> id="g3973"> </ins> <path <del> sodipodi:type="arc" style="fill:none;stroke:#ffffff;stroke-width:162.01495361;stroke-linejoin:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" id="path3847" sodipodi:cx="8" sodipodi:cy="432.36218" sodipodi:rx="720" sodipodi:ry="720" d="M 728,432.36218 C 728,830.0072 405.64502,1152.3622 8,1152.3622 c -397.64502,0 -720,-322.355 -720,-720.00002 0,-397.645017 322.35498,-720 720,-720 397.64502,0 720,322.354983 720,720 z" transform="matrix(-0.05312297,-0.23127578,0.23112783,-0.05247783,269.21748,562.56174)" /> </del> <ins> inkscape:export-ydpi="90" inkscape:export-xdpi="90" transform="matrix(-0.70423675,-1.1472157,1.1566815,-0.69847357,-57.760066,-127.50052)" inkscape:transform-center-y="-88.166574" inkscape:transform-center-x="85.453542" d="m -391.42855,129.50504 -168.04608,108.95128 -189.95655,-63.45371 -68.82586,-188.076755 104.1321,-171.074165 198.67647,-25.24924 143.61341,139.588871 z" inkscape:randomized="0" inkscape:rounded="0" inkscape:flatsided="true" sodipodi:arg2="0.99558779" sodipodi:arg1="0.54678884" sodipodi:r2="207.9371" sodipodi:r1="230.79279" sodipodi:cy="9.5050402" sodipodi:cx="-588.57141" sodipodi:sides="7" id="path3054" style="fill:#326de6;fill-opacity:1;stroke:#326de6;stroke-width:69;stroke-linejoin:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" sodipodi:type="star" /> </ins> <g <del> id="g5927"> <g id="g4131" transform="matrix(1,0,0,0.93883617,3.0146411,22.388286)"> <g transform="matrix(0.27453777,-3.262896e-5,2.8624424e-5,0.31011525,143.53725,308.78719)" id="g3892-2" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-opacity:1"> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" d="m 751.73724,103.83546 68.45129,0.15616 4.29438,474.38067 -101.29515,0.88337 z" id="rect3851-5" inkscape:connector-curvature="0" sodipodi:nodetypes="ccccc" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" d="m 866.65544,103.83523 -68.45129,0.15616 -4.29439,474.38067 101.29515,0.88337 z" id="rect3851-4-35" inkscape:connector-curvature="0" sodipodi:nodetypes="ccccc" /> </g> <g id="g3303"> <path sodipodi:type="arc" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.07997656;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" id="path3896-5" sodipodi:cx="809.41376" sodipodi:cy="-99.897148" sodipodi:rx="51.178402" sodipodi:ry="51.178402" d="m 860.59216,-99.897148 c 0,28.265051 -22.91335,51.178402 -51.1784,51.178402 -28.26505,0 -51.1784,-22.913351 -51.1784,-51.178402 0,-28.265052 22.91335,-51.178402 51.1784,-51.178402 28.26505,0 51.1784,22.91335 51.1784,51.178402 z" transform="matrix(0.28575795,-3.1068327e-5,3.4122922e-5,0.33818241,134.39948,319.2262)" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 351.07606,285.18762 c 0.001,1.36972 -0.0624,3.33785 -0.0163,4.65708 0.19262,5.5132 1.32752,9.74051 2.00662,14.81995 1.23028,10.87161 2.26477,19.8822 1.6286,28.25841 -0.57783,4.19204 -2.85742,5.84225 -4.76113,7.78258 l 23.06717,16.25738 -3.46339,-71.65344 z" id="path3898-37" inkscape:connector-curvature="0" sodipodi:nodetypes="cscccccc" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 380.31469,285.18762 c -10e-4,1.36972 0.0624,3.33785 0.0163,4.65708 -0.19262,5.5132 -1.32752,9.74051 -2.00662,14.81995 -1.23028,10.87161 -2.26477,19.8822 -1.6286,28.25841 0.57783,4.19204 2.85742,5.84225 4.76113,7.78258 l -23.06717,16.25738 3.46339,-71.65344 z" id="path3898-37-3" inkscape:connector-curvature="0" sodipodi:nodetypes="cscccccc" /> </g> </g> </del> <ins> transform="translate(-0.97949518,-0.04249773)" id="g6172"> </ins> <path <del> sodipodi:type="star" style="fill:none;stroke:#ffffff;stroke-width:38;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" id="path4840" sodipodi:sides="7" sodipodi:cx="253.5687" sodipodi:cy="480.14536" sodipodi:r1="48.216618" sodipodi:r2="43.441673" sodipodi:arg1="1.3089969" sodipodi:arg2="1.7577959" inkscape:flatsided="true" inkscape:rounded="0" inkscape:randomized="0" d="m 266.04808,526.71903 -41.11138,-7.77869 -19.5509,-36.99211 16.7318,-38.34971 40.41512,-10.8292 33.66503,24.84592 1.56448,41.81155 z" inkscape:transform-center-x="-0.0016128867" inkscape:transform-center-y="2.371582" transform="matrix(0.80325602,-0.59012374,0.59264431,0.79983971,-119.57145,303.63055)" /> </del> <ins> transform="matrix(-0.05312297,-0.23127578,0.23112783,-0.05247783,269.21748,562.56174)" d="M 728,432.36218 C 728,830.0072 405.64502,1152.3622 8,1152.3622 c -397.64502,0 -720,-322.355 -720,-720.00002 0,-397.645017 322.35498,-720 720,-720 397.64502,0 720,322.354983 720,720 z" sodipodi:ry="720" sodipodi:rx="720" sodipodi:cy="432.36218" sodipodi:cx="8" id="path3847" style="fill:none;stroke:#ffffff;stroke-width:162.01495361;stroke-linejoin:round;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" sodipodi:type="arc" /> </ins> <g <del> id="g4131-6" transform="matrix(0.62348395,-0.78183615,0.73401606,0.58534928,-262.59282,502.41369)"> </del> <ins> id="g5927"> </ins> <g <del> transform="matrix(0.27453777,-3.262896e-5,2.8624424e-5,0.31011525,143.53725,308.78719)" id="g3892-2-4" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-opacity:1"> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" d="m 751.73724,103.83546 68.45129,0.15616 4.29438,474.38067 -101.29515,0.88337 z" id="rect3851-5-1" inkscape:connector-curvature="0" sodipodi:nodetypes="ccccc" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" d="m 866.65544,103.83523 -68.45129,0.15616 -4.29439,474.38067 101.29515,0.88337 z" id="rect3851-4-35-7" inkscape:connector-curvature="0" sodipodi:nodetypes="ccccc" /> </del> <ins> transform="matrix(1,0,0,0.93883617,3.0146411,22.388286)" id="g4131"> <g style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-opacity:1" id="g3892-2" transform="matrix(0.27453777,-3.262896e-5,2.8624424e-5,0.31011525,143.53725,308.78719)"> <path sodipodi:nodetypes="ccccc" inkscape:connector-curvature="0" id="rect3851-5" d="m 751.73724,103.83546 68.45129,0.15616 4.29438,474.38067 -101.29515,0.88337 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" /> <path sodipodi:nodetypes="ccccc" inkscape:connector-curvature="0" id="rect3851-4-35" d="m 866.65544,103.83523 -68.45129,0.15616 -4.29439,474.38067 101.29515,0.88337 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" /> </g> <g id="g3303"> <path transform="matrix(0.28575795,-3.1068327e-5,3.4122922e-5,0.33818241,134.39948,319.2262)" d="m 860.59216,-99.897148 c 0,28.265051 -22.91335,51.178402 -51.1784,51.178402 -28.26505,0 -51.1784,-22.913351 -51.1784,-51.178402 0,-28.265052 22.91335,-51.178402 51.1784,-51.178402 28.26505,0 51.1784,22.91335 51.1784,51.178402 z" sodipodi:ry="51.178402" sodipodi:rx="51.178402" sodipodi:cy="-99.897148" sodipodi:cx="809.41376" id="path3896-5" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.07997656;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" sodipodi:type="arc" /> <path sodipodi:nodetypes="cscccccc" inkscape:connector-curvature="0" id="path3898-37" d="m 351.07606,285.18762 c 0.001,1.36972 -0.0624,3.33785 -0.0163,4.65708 0.19262,5.5132 1.32752,9.74051 2.00662,14.81995 1.23028,10.87161 2.26477,19.8822 1.6286,28.25841 -0.57783,4.19204 -2.85742,5.84225 -4.76113,7.78258 l 23.06717,16.25738 -3.46339,-71.65344 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" /> <path sodipodi:nodetypes="cscccccc" inkscape:connector-curvature="0" id="path3898-37-3" d="m 380.31469,285.18762 c -10e-4,1.36972 0.0624,3.33785 0.0163,4.65708 -0.19262,5.5132 -1.32752,9.74051 -2.00662,14.81995 -1.23028,10.87161 -2.26477,19.8822 -1.6286,28.25841 0.57783,4.19204 2.85742,5.84225 4.76113,7.78258 l -23.06717,16.25738 3.46339,-71.65344 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" /> </g> </ins> </g> <ins> <path transform="matrix(0.80325602,-0.59012374,0.59264431,0.79983971,-119.57145,303.63055)" inkscape:transform-center-y="2.371582" inkscape:transform-center-x="-0.0016128867" d="m 266.04808,526.71903 -41.11138,-7.77869 -19.5509,-36.99211 16.7318,-38.34971 40.41512,-10.8292 33.66503,24.84592 1.56448,41.81155 z" inkscape:randomized="0" inkscape:rounded="0" inkscape:flatsided="true" sodipodi:arg2="1.7577959" sodipodi:arg1="1.3089969" sodipodi:r2="43.441673" sodipodi:r1="48.216618" sodipodi:cy="480.14536" sodipodi:cx="253.5687" sodipodi:sides="7" id="path4840" style="fill:none;stroke:#ffffff;stroke-width:38;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" sodipodi:type="star" /> </ins> <g <del> id="g3303-0"> <path sodipodi:type="arc" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.07997656;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" id="path3896-5-0" sodipodi:cx="809.41376" sodipodi:cy="-99.897148" sodipodi:rx="51.178402" sodipodi:ry="51.178402" d="m 860.59216,-99.897148 c 0,28.265051 -22.91335,51.178402 -51.1784,51.178402 -28.26505,0 -51.1784,-22.913351 -51.1784,-51.178402 0,-28.265052 22.91335,-51.178402 51.1784,-51.178402 28.26505,0 51.1784,22.91335 51.1784,51.178402 z" transform="matrix(0.28575795,-3.1068327e-5,3.4122922e-5,0.33818241,134.39948,319.2262)" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 351.07606,285.18762 c 0.001,1.36972 -0.0624,3.33785 -0.0163,4.65708 0.19262,5.5132 1.32752,9.74051 2.00662,14.81995 1.23028,10.87161 2.26477,19.8822 1.6286,28.25841 -0.57783,4.19204 -2.85742,5.84225 -4.76113,7.78258 l 23.06717,16.25738 -3.46339,-71.65344 z" id="path3898-37-5" inkscape:connector-curvature="0" sodipodi:nodetypes="cscccccc" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 380.31469,285.18762 c -10e-4,1.36972 0.0624,3.33785 0.0163,4.65708 -0.19262,5.5132 -1.32752,9.74051 -2.00662,14.81995 -1.23028,10.87161 -2.26477,19.8822 -1.6286,28.25841 0.57783,4.19204 2.85742,5.84225 4.76113,7.78258 l -23.06717,16.25738 3.46339,-71.65344 z" id="path3898-37-3-0" inkscape:connector-curvature="0" sodipodi:nodetypes="cscccccc" /> </del> <ins> transform="matrix(0.62348395,-0.78183615,0.73401606,0.58534928,-262.59282,502.41369)" id="g4131-6"> <g style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-opacity:1" id="g3892-2-4" transform="matrix(0.27453777,-3.262896e-5,2.8624424e-5,0.31011525,143.53725,308.78719)"> <path sodipodi:nodetypes="ccccc" inkscape:connector-curvature="0" id="rect3851-5-1" d="m 751.73724,103.83546 68.45129,0.15616 4.29438,474.38067 -101.29515,0.88337 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" /> <path sodipodi:nodetypes="ccccc" inkscape:connector-curvature="0" id="rect3851-4-35-7" d="m 866.65544,103.83523 -68.45129,0.15616 -4.29439,474.38067 101.29515,0.88337 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" /> </g> <g id="g3303-0"> <path transform="matrix(0.28575795,-3.1068327e-5,3.4122922e-5,0.33818241,134.39948,319.2262)" d="m 860.59216,-99.897148 c 0,28.265051 -22.91335,51.178402 -51.1784,51.178402 -28.26505,0 -51.1784,-22.913351 -51.1784,-51.178402 0,-28.265052 22.91335,-51.178402 51.1784,-51.178402 28.26505,0 51.1784,22.91335 51.1784,51.178402 z" sodipodi:ry="51.178402" sodipodi:rx="51.178402" sodipodi:cy="-99.897148" sodipodi:cx="809.41376" id="path3896-5-0" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.07997656;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" sodipodi:type="arc" /> <path sodipodi:nodetypes="cscccccc" inkscape:connector-curvature="0" id="path3898-37-5" d="m 351.07606,285.18762 c 0.001,1.36972 -0.0624,3.33785 -0.0163,4.65708 0.19262,5.5132 1.32752,9.74051 2.00662,14.81995 1.23028,10.87161 2.26477,19.8822 1.6286,28.25841 -0.57783,4.19204 -2.85742,5.84225 -4.76113,7.78258 l 23.06717,16.25738 -3.46339,-71.65344 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" /> <path sodipodi:nodetypes="cscccccc" inkscape:connector-curvature="0" id="path3898-37-3-0" d="m 380.31469,285.18762 c -10e-4,1.36972 0.0624,3.33785 0.0163,4.65708 -0.19262,5.5132 -1.32752,9.74051 -2.00662,14.81995 -1.23028,10.87161 -2.26477,19.8822 -1.6286,28.25841 0.57783,4.19204 2.85742,5.84225 4.76113,7.78258 l -23.06717,16.25738 3.46339,-71.65344 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" /> </g> </ins> </g> <del> </g> <g id="g4131-8" transform="matrix(-0.22253552,-0.97492458,0.91529446,-0.2089244,-52.601372,1009.4621)"> </del> <g <del> transform="matrix(0.27453777,-3.262896e-5,2.8624424e-5,0.31011525,143.53725,308.78719)" id="g3892-2-5" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-opacity:1"> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" d="m 751.73724,103.83546 68.45129,0.15616 4.29438,474.38067 -101.29515,0.88337 z" id="rect3851-5-7" inkscape:connector-curvature="0" sodipodi:nodetypes="ccccc" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" d="m 866.65544,103.83523 -68.45129,0.15616 -4.29439,474.38067 101.29515,0.88337 z" id="rect3851-4-35-5" inkscape:connector-curvature="0" sodipodi:nodetypes="ccccc" /> </del> <ins> transform="matrix(-0.22253552,-0.97492458,0.91529446,-0.2089244,-52.601372,1009.4621)" id="g4131-8"> <g style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-opacity:1" id="g3892-2-5" transform="matrix(0.27453777,-3.262896e-5,2.8624424e-5,0.31011525,143.53725,308.78719)"> <path sodipodi:nodetypes="ccccc" inkscape:connector-curvature="0" id="rect3851-5-7" d="m 751.73724,103.83546 68.45129,0.15616 4.29438,474.38067 -101.29515,0.88337 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" /> <path sodipodi:nodetypes="ccccc" inkscape:connector-curvature="0" id="rect3851-4-35-5" d="m 866.65544,103.83523 -68.45129,0.15616 -4.29439,474.38067 101.29515,0.88337 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" /> </g> <g id="g3303-4"> <path transform="matrix(0.28575795,-3.1068327e-5,3.4122922e-5,0.33818241,134.39948,319.2262)" d="m 860.59216,-99.897148 c 0,28.265051 -22.91335,51.178402 -51.1784,51.178402 -28.26505,0 -51.1784,-22.913351 -51.1784,-51.178402 0,-28.265052 22.91335,-51.178402 51.1784,-51.178402 28.26505,0 51.1784,22.91335 51.1784,51.178402 z" sodipodi:ry="51.178402" sodipodi:rx="51.178402" sodipodi:cy="-99.897148" sodipodi:cx="809.41376" id="path3896-5-2" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.07997656;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" sodipodi:type="arc" /> <path sodipodi:nodetypes="cscccccc" inkscape:connector-curvature="0" id="path3898-37-7" d="m 351.07606,285.18762 c 0.001,1.36972 -0.0624,3.33785 -0.0163,4.65708 0.19262,5.5132 1.32752,9.74051 2.00662,14.81995 1.23028,10.87161 2.26477,19.8822 1.6286,28.25841 -0.57783,4.19204 -2.85742,5.84225 -4.76113,7.78258 l 23.06717,16.25738 -3.46339,-71.65344 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" /> <path sodipodi:nodetypes="cscccccc" inkscape:connector-curvature="0" id="path3898-37-3-5" d="m 380.31469,285.18762 c -10e-4,1.36972 0.0624,3.33785 0.0163,4.65708 -0.19262,5.5132 -1.32752,9.74051 -2.00662,14.81995 -1.23028,10.87161 -2.26477,19.8822 -1.6286,28.25841 0.57783,4.19204 2.85742,5.84225 4.76113,7.78258 l -23.06717,16.25738 3.46339,-71.65344 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" /> </g> </ins> </g> <g <del> id="g3303-4"> <path sodipodi:type="arc" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.07997656;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" id="path3896-5-2" sodipodi:cx="809.41376" sodipodi:cy="-99.897148" sodipodi:rx="51.178402" sodipodi:ry="51.178402" d="m 860.59216,-99.897148 c 0,28.265051 -22.91335,51.178402 -51.1784,51.178402 -28.26505,0 -51.1784,-22.913351 -51.1784,-51.178402 0,-28.265052 22.91335,-51.178402 51.1784,-51.178402 28.26505,0 51.1784,22.91335 51.1784,51.178402 z" transform="matrix(0.28575795,-3.1068327e-5,3.4122922e-5,0.33818241,134.39948,319.2262)" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 351.07606,285.18762 c 0.001,1.36972 -0.0624,3.33785 -0.0163,4.65708 0.19262,5.5132 1.32752,9.74051 2.00662,14.81995 1.23028,10.87161 2.26477,19.8822 1.6286,28.25841 -0.57783,4.19204 -2.85742,5.84225 -4.76113,7.78258 l 23.06717,16.25738 -3.46339,-71.65344 z" id="path3898-37-7" inkscape:connector-curvature="0" sodipodi:nodetypes="cscccccc" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 380.31469,285.18762 c -10e-4,1.36972 0.0624,3.33785 0.0163,4.65708 -0.19262,5.5132 -1.32752,9.74051 -2.00662,14.81995 -1.23028,10.87161 -2.26477,19.8822 -1.6286,28.25841 0.57783,4.19204 2.85742,5.84225 4.76113,7.78258 l -23.06717,16.25738 3.46339,-71.65344 z" id="path3898-37-3-5" inkscape:connector-curvature="0" sodipodi:nodetypes="cscccccc" /> </g> </g> <g id="g4131-6-8" transform="matrix(-0.90097861,-0.43386351,0.40732676,-0.84587131,474.49429,1161.5866)"> <g transform="matrix(0.27453777,-3.262896e-5,2.8624424e-5,0.31011525,143.53725,308.78719)" id="g3892-2-4-6" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-opacity:1"> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" d="m 751.73724,103.83546 68.45129,0.15616 4.29438,474.38067 -101.29515,0.88337 z" id="rect3851-5-1-8" inkscape:connector-curvature="0" sodipodi:nodetypes="ccccc" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" d="m 866.65544,103.83523 -68.45129,0.15616 -4.29439,474.38067 101.29515,0.88337 z" id="rect3851-4-35-7-3" inkscape:connector-curvature="0" sodipodi:nodetypes="ccccc" /> </del> <ins> transform="matrix(-0.90097861,-0.43386351,0.40732676,-0.84587131,474.49429,1161.5866)" id="g4131-6-8"> <g style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-opacity:1" id="g3892-2-4-6" transform="matrix(0.27453777,-3.262896e-5,2.8624424e-5,0.31011525,143.53725,308.78719)"> <path sodipodi:nodetypes="ccccc" inkscape:connector-curvature="0" id="rect3851-5-1-8" d="m 751.73724,103.83546 68.45129,0.15616 4.29438,474.38067 -101.29515,0.88337 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" /> <path sodipodi:nodetypes="ccccc" inkscape:connector-curvature="0" id="rect3851-4-35-7-3" d="m 866.65544,103.83523 -68.45129,0.15616 -4.29439,474.38067 101.29515,0.88337 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" /> </g> <g id="g3303-0-7"> <path transform="matrix(0.28575795,-3.1068327e-5,3.4122922e-5,0.33818241,134.39948,319.2262)" d="m 860.59216,-99.897148 c 0,28.265051 -22.91335,51.178402 -51.1784,51.178402 -28.26505,0 -51.1784,-22.913351 -51.1784,-51.178402 0,-28.265052 22.91335,-51.178402 51.1784,-51.178402 28.26505,0 51.1784,22.91335 51.1784,51.178402 z" sodipodi:ry="51.178402" sodipodi:rx="51.178402" sodipodi:cy="-99.897148" sodipodi:cx="809.41376" id="path3896-5-0-4" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.07997656;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" sodipodi:type="arc" /> <path sodipodi:nodetypes="cscccccc" inkscape:connector-curvature="0" id="path3898-37-5-6" d="m 351.07606,285.18762 c 0.001,1.36972 -0.0624,3.33785 -0.0163,4.65708 0.19262,5.5132 1.32752,9.74051 2.00662,14.81995 1.23028,10.87161 2.26477,19.8822 1.6286,28.25841 -0.57783,4.19204 -2.85742,5.84225 -4.76113,7.78258 l 23.06717,16.25738 -3.46339,-71.65344 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" /> <path sodipodi:nodetypes="cscccccc" inkscape:connector-curvature="0" id="path3898-37-3-0-4" d="m 380.31469,285.18762 c -10e-4,1.36972 0.0624,3.33785 0.0163,4.65708 -0.19262,5.5132 -1.32752,9.74051 -2.00662,14.81995 -1.23028,10.87161 -2.26477,19.8822 -1.6286,28.25841 0.57783,4.19204 2.85742,5.84225 4.76113,7.78258 l -23.06717,16.25738 3.46339,-71.65344 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" /> </g> </ins> </g> <g <del> id="g3303-0-7"> <path sodipodi:type="arc" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.07997656;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" id="path3896-5-0-4" sodipodi:cx="809.41376" sodipodi:cy="-99.897148" sodipodi:rx="51.178402" sodipodi:ry="51.178402" d="m 860.59216,-99.897148 c 0,28.265051 -22.91335,51.178402 -51.1784,51.178402 -28.26505,0 -51.1784,-22.913351 -51.1784,-51.178402 0,-28.265052 22.91335,-51.178402 51.1784,-51.178402 28.26505,0 51.1784,22.91335 51.1784,51.178402 z" transform="matrix(0.28575795,-3.1068327e-5,3.4122922e-5,0.33818241,134.39948,319.2262)" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 351.07606,285.18762 c 0.001,1.36972 -0.0624,3.33785 -0.0163,4.65708 0.19262,5.5132 1.32752,9.74051 2.00662,14.81995 1.23028,10.87161 2.26477,19.8822 1.6286,28.25841 -0.57783,4.19204 -2.85742,5.84225 -4.76113,7.78258 l 23.06717,16.25738 -3.46339,-71.65344 z" id="path3898-37-5-6" inkscape:connector-curvature="0" sodipodi:nodetypes="cscccccc" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 380.31469,285.18762 c -10e-4,1.36972 0.0624,3.33785 0.0163,4.65708 -0.19262,5.5132 -1.32752,9.74051 -2.00662,14.81995 -1.23028,10.87161 -2.26477,19.8822 -1.6286,28.25841 0.57783,4.19204 2.85742,5.84225 4.76113,7.78258 l -23.06717,16.25738 3.46339,-71.65344 z" id="path3898-37-3-0-4" inkscape:connector-curvature="0" sodipodi:nodetypes="cscccccc" /> </del> <ins> transform="matrix(-0.90095588,0.4339107,-0.40737106,-0.84584997,921.98234,844.00136)" id="g4131-6-2"> <g style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-opacity:1" id="g3892-2-4-4" transform="matrix(0.27453777,-3.262896e-5,2.8624424e-5,0.31011525,143.53725,308.78719)"> <path sodipodi:nodetypes="ccccc" inkscape:connector-curvature="0" id="rect3851-5-1-3" d="m 751.73724,103.83546 68.45129,0.15616 4.29438,474.38067 -101.29515,0.88337 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" /> <path sodipodi:nodetypes="ccccc" inkscape:connector-curvature="0" id="rect3851-4-35-7-4" d="m 866.65544,103.83523 -68.45129,0.15616 -4.29439,474.38067 101.29515,0.88337 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" /> </g> <g id="g3303-0-4"> <path transform="matrix(0.28575795,-3.1068327e-5,3.4122922e-5,0.33818241,134.39948,319.2262)" d="m 860.59216,-99.897148 c 0,28.265051 -22.91335,51.178402 -51.1784,51.178402 -28.26505,0 -51.1784,-22.913351 -51.1784,-51.178402 0,-28.265052 22.91335,-51.178402 51.1784,-51.178402 28.26505,0 51.1784,22.91335 51.1784,51.178402 z" sodipodi:ry="51.178402" sodipodi:rx="51.178402" sodipodi:cy="-99.897148" sodipodi:cx="809.41376" id="path3896-5-0-0" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.07997656;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" sodipodi:type="arc" /> <path sodipodi:nodetypes="cscccccc" inkscape:connector-curvature="0" id="path3898-37-5-1" d="m 351.07606,285.18762 c 0.001,1.36972 -0.0624,3.33785 -0.0163,4.65708 0.19262,5.5132 1.32752,9.74051 2.00662,14.81995 1.23028,10.87161 2.26477,19.8822 1.6286,28.25841 -0.57783,4.19204 -2.85742,5.84225 -4.76113,7.78258 l 23.06717,16.25738 -3.46339,-71.65344 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" /> <path sodipodi:nodetypes="cscccccc" inkscape:connector-curvature="0" id="path3898-37-3-0-9" d="m 380.31469,285.18762 c -10e-4,1.36972 0.0624,3.33785 0.0163,4.65708 -0.19262,5.5132 -1.32752,9.74051 -2.00662,14.81995 -1.23028,10.87161 -2.26477,19.8822 -1.6286,28.25841 0.57783,4.19204 2.85742,5.84225 4.76113,7.78258 l -23.06717,16.25738 3.46339,-71.65344 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" /> </g> </ins> </g> <del> </g> <g id="g4131-6-2" transform="matrix(-0.90095588,0.4339107,-0.40737106,-0.84584997,921.98234,844.00136)"> </del> <g <del> transform="matrix(0.27453777,-3.262896e-5,2.8624424e-5,0.31011525,143.53725,308.78719)" id="g3892-2-4-4" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-opacity:1"> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" d="m 751.73724,103.83546 68.45129,0.15616 4.29438,474.38067 -101.29515,0.88337 z" id="rect3851-5-1-3" inkscape:connector-curvature="0" sodipodi:nodetypes="ccccc" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" d="m 866.65544,103.83523 -68.45129,0.15616 -4.29439,474.38067 101.29515,0.88337 z" id="rect3851-4-35-7-4" inkscape:connector-curvature="0" sodipodi:nodetypes="ccccc" /> </del> <ins> transform="matrix(-0.22248447,0.97493623,-0.9153054,-0.20887646,952.77435,296.05397)" id="g4131-8-4"> <g style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-opacity:1" id="g3892-2-5-6" transform="matrix(0.27453777,-3.262896e-5,2.8624424e-5,0.31011525,143.53725,308.78719)"> <path sodipodi:nodetypes="ccccc" inkscape:connector-curvature="0" id="rect3851-5-7-0" d="m 751.73724,103.83546 68.45129,0.15616 4.29438,474.38067 -101.29515,0.88337 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" /> <path sodipodi:nodetypes="ccccc" inkscape:connector-curvature="0" id="rect3851-4-35-5-4" d="m 866.65544,103.83523 -68.45129,0.15616 -4.29439,474.38067 101.29515,0.88337 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" /> </g> <g id="g3303-4-3"> <path transform="matrix(0.28575795,-3.1068327e-5,3.4122922e-5,0.33818241,134.39948,319.2262)" d="m 860.59216,-99.897148 c 0,28.265051 -22.91335,51.178402 -51.1784,51.178402 -28.26505,0 -51.1784,-22.913351 -51.1784,-51.178402 0,-28.265052 22.91335,-51.178402 51.1784,-51.178402 28.26505,0 51.1784,22.91335 51.1784,51.178402 z" sodipodi:ry="51.178402" sodipodi:rx="51.178402" sodipodi:cy="-99.897148" sodipodi:cx="809.41376" id="path3896-5-2-2" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.07997656;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" sodipodi:type="arc" /> <path sodipodi:nodetypes="cscccccc" inkscape:connector-curvature="0" id="path3898-37-7-0" d="m 351.07606,285.18762 c 0.001,1.36972 -0.0624,3.33785 -0.0163,4.65708 0.19262,5.5132 1.32752,9.74051 2.00662,14.81995 1.23028,10.87161 2.26477,19.8822 1.6286,28.25841 -0.57783,4.19204 -2.85742,5.84225 -4.76113,7.78258 l 23.06717,16.25738 -3.46339,-71.65344 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" /> <path sodipodi:nodetypes="cscccccc" inkscape:connector-curvature="0" id="path3898-37-3-5-9" d="m 380.31469,285.18762 c -10e-4,1.36972 0.0624,3.33785 0.0163,4.65708 -0.19262,5.5132 -1.32752,9.74051 -2.00662,14.81995 -1.23028,10.87161 -2.26477,19.8822 -1.6286,28.25841 0.57783,4.19204 2.85742,5.84225 4.76113,7.78258 l -23.06717,16.25738 3.46339,-71.65344 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" /> </g> </ins> </g> <g <del> id="g3303-0-4"> <path sodipodi:type="arc" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.07997656;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" id="path3896-5-0-0" sodipodi:cx="809.41376" sodipodi:cy="-99.897148" sodipodi:rx="51.178402" sodipodi:ry="51.178402" d="m 860.59216,-99.897148 c 0,28.265051 -22.91335,51.178402 -51.1784,51.178402 -28.26505,0 -51.1784,-22.913351 -51.1784,-51.178402 0,-28.265052 22.91335,-51.178402 51.1784,-51.178402 28.26505,0 51.1784,22.91335 51.1784,51.178402 z" transform="matrix(0.28575795,-3.1068327e-5,3.4122922e-5,0.33818241,134.39948,319.2262)" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 351.07606,285.18762 c 0.001,1.36972 -0.0624,3.33785 -0.0163,4.65708 0.19262,5.5132 1.32752,9.74051 2.00662,14.81995 1.23028,10.87161 2.26477,19.8822 1.6286,28.25841 -0.57783,4.19204 -2.85742,5.84225 -4.76113,7.78258 l 23.06717,16.25738 -3.46339,-71.65344 z" id="path3898-37-5-1" inkscape:connector-curvature="0" sodipodi:nodetypes="cscccccc" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 380.31469,285.18762 c -10e-4,1.36972 0.0624,3.33785 0.0163,4.65708 -0.19262,5.5132 -1.32752,9.74051 -2.00662,14.81995 -1.23028,10.87161 -2.26477,19.8822 -1.6286,28.25841 0.57783,4.19204 2.85742,5.84225 4.76113,7.78258 l -23.06717,16.25738 3.46339,-71.65344 z" id="path3898-37-3-0-9" inkscape:connector-curvature="0" sodipodi:nodetypes="cscccccc" /> </del> <ins> transform="matrix(0.6235249,0.78180349,-0.7339854,0.58538773,543.87371,-69.694532)" id="g4131-6-8-9"> <g style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-opacity:1" id="g3892-2-4-6-1" transform="matrix(0.27453777,-3.262896e-5,2.8624424e-5,0.31011525,143.53725,308.78719)"> <path sodipodi:nodetypes="ccccc" inkscape:connector-curvature="0" id="rect3851-5-1-8-0" d="m 751.73724,103.83546 68.45129,0.15616 4.29438,474.38067 -101.29515,0.88337 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" /> <path sodipodi:nodetypes="ccccc" inkscape:connector-curvature="0" id="rect3851-4-35-7-3-9" d="m 866.65544,103.83523 -68.45129,0.15616 -4.29439,474.38067 101.29515,0.88337 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" /> </g> <g id="g3303-0-7-8"> <path transform="matrix(0.28575795,-3.1068327e-5,3.4122922e-5,0.33818241,134.39948,319.2262)" d="m 860.59216,-99.897148 c 0,28.265051 -22.91335,51.178402 -51.1784,51.178402 -28.26505,0 -51.1784,-22.913351 -51.1784,-51.178402 0,-28.265052 22.91335,-51.178402 51.1784,-51.178402 28.26505,0 51.1784,22.91335 51.1784,51.178402 z" sodipodi:ry="51.178402" sodipodi:rx="51.178402" sodipodi:cy="-99.897148" sodipodi:cx="809.41376" id="path3896-5-0-4-9" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.07997656;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" sodipodi:type="arc" /> <path sodipodi:nodetypes="cscccccc" inkscape:connector-curvature="0" id="path3898-37-5-6-0" d="m 351.07606,285.18762 c 0.001,1.36972 -0.0624,3.33785 -0.0163,4.65708 0.19262,5.5132 1.32752,9.74051 2.00662,14.81995 1.23028,10.87161 2.26477,19.8822 1.6286,28.25841 -0.57783,4.19204 -2.85742,5.84225 -4.76113,7.78258 l 23.06717,16.25738 -3.46339,-71.65344 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" /> <path sodipodi:nodetypes="cscccccc" inkscape:connector-curvature="0" id="path3898-37-3-0-4-4" d="m 380.31469,285.18762 c -10e-4,1.36972 0.0624,3.33785 0.0163,4.65708 -0.19262,5.5132 -1.32752,9.74051 -2.00662,14.81995 -1.23028,10.87161 -2.26477,19.8822 -1.6286,28.25841 0.57783,4.19204 2.85742,5.84225 4.76113,7.78258 l -23.06717,16.25738 3.46339,-71.65344 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" /> </g> </ins> </g> </g> <g <del> id="g4131-8-4" transform="matrix(-0.22248447,0.97493623,-0.9153054,-0.20887646,952.77435,296.05397)"> <g transform="matrix(0.27453777,-3.262896e-5,2.8624424e-5,0.31011525,143.53725,308.78719)" id="g3892-2-5-6" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-opacity:1"> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" d="m 751.73724,103.83546 68.45129,0.15616 4.29438,474.38067 -101.29515,0.88337 z" id="rect3851-5-7-0" inkscape:connector-curvature="0" sodipodi:nodetypes="ccccc" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" d="m 866.65544,103.83523 -68.45129,0.15616 -4.29439,474.38067 101.29515,0.88337 z" id="rect3851-4-35-5-4" inkscape:connector-curvature="0" sodipodi:nodetypes="ccccc" /> </g> <g id="g3303-4-3"> <path sodipodi:type="arc" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.07997656;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" id="path3896-5-2-2" sodipodi:cx="809.41376" sodipodi:cy="-99.897148" sodipodi:rx="51.178402" sodipodi:ry="51.178402" d="m 860.59216,-99.897148 c 0,28.265051 -22.91335,51.178402 -51.1784,51.178402 -28.26505,0 -51.1784,-22.913351 -51.1784,-51.178402 0,-28.265052 22.91335,-51.178402 51.1784,-51.178402 28.26505,0 51.1784,22.91335 51.1784,51.178402 z" transform="matrix(0.28575795,-3.1068327e-5,3.4122922e-5,0.33818241,134.39948,319.2262)" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 351.07606,285.18762 c 0.001,1.36972 -0.0624,3.33785 -0.0163,4.65708 0.19262,5.5132 1.32752,9.74051 2.00662,14.81995 1.23028,10.87161 2.26477,19.8822 1.6286,28.25841 -0.57783,4.19204 -2.85742,5.84225 -4.76113,7.78258 l 23.06717,16.25738 -3.46339,-71.65344 z" id="path3898-37-7-0" inkscape:connector-curvature="0" sodipodi:nodetypes="cscccccc" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 380.31469,285.18762 c -10e-4,1.36972 0.0624,3.33785 0.0163,4.65708 -0.19262,5.5132 -1.32752,9.74051 -2.00662,14.81995 -1.23028,10.87161 -2.26477,19.8822 -1.6286,28.25841 0.57783,4.19204 2.85742,5.84225 4.76113,7.78258 l -23.06717,16.25738 3.46339,-71.65344 z" id="path3898-37-3-5-9" inkscape:connector-curvature="0" sodipodi:nodetypes="cscccccc" /> </g> </g> <g id="g4131-6-8-9" transform="matrix(0.6235249,0.78180349,-0.7339854,0.58538773,543.87371,-69.694532)"> <g transform="matrix(0.27453777,-3.262896e-5,2.8624424e-5,0.31011525,143.53725,308.78719)" id="g3892-2-4-6-1" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-opacity:1"> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" d="m 751.73724,103.83546 68.45129,0.15616 4.29438,474.38067 -101.29515,0.88337 z" id="rect3851-5-1-8-0" inkscape:connector-curvature="0" sodipodi:nodetypes="ccccc" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.76751065;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1" d="m 866.65544,103.83523 -68.45129,0.15616 -4.29439,474.38067 101.29515,0.88337 z" id="rect3851-4-35-7-3-9" inkscape:connector-curvature="0" sodipodi:nodetypes="ccccc" /> </g> <g id="g3303-0-7-8"> <path sodipodi:type="arc" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:1.07997656;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" id="path3896-5-0-4-9" sodipodi:cx="809.41376" sodipodi:cy="-99.897148" sodipodi:rx="51.178402" sodipodi:ry="51.178402" d="m 860.59216,-99.897148 c 0,28.265051 -22.91335,51.178402 -51.1784,51.178402 -28.26505,0 -51.1784,-22.913351 -51.1784,-51.178402 0,-28.265052 22.91335,-51.178402 51.1784,-51.178402 28.26505,0 51.1784,22.91335 51.1784,51.178402 z" transform="matrix(0.28575795,-3.1068327e-5,3.4122922e-5,0.33818241,134.39948,319.2262)" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 351.07606,285.18762 c 0.001,1.36972 -0.0624,3.33785 -0.0163,4.65708 0.19262,5.5132 1.32752,9.74051 2.00662,14.81995 1.23028,10.87161 2.26477,19.8822 1.6286,28.25841 -0.57783,4.19204 -2.85742,5.84225 -4.76113,7.78258 l 23.06717,16.25738 -3.46339,-71.65344 z" id="path3898-37-5-6-0" inkscape:connector-curvature="0" sodipodi:nodetypes="cscccccc" /> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0.33545998;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 380.31469,285.18762 c -10e-4,1.36972 0.0624,3.33785 0.0163,4.65708 -0.19262,5.5132 -1.32752,9.74051 -2.00662,14.81995 -1.23028,10.87161 -2.26477,19.8822 -1.6286,28.25841 0.57783,4.19204 2.85742,5.84225 4.76113,7.78258 l -23.06717,16.25738 3.46339,-71.65344 z" id="path3898-37-3-0-4-4" inkscape:connector-curvature="0" sodipodi:nodetypes="cscccccc" /> </g> </del> <ins> id="g6152"> <path inkscape:connector-curvature="0" id="rect6068" d="m 345.7441,464.40625 c -0.29134,6.86335 -5.93996,12.34375 -12.875,12.34375 -2.84081,0 -5.46294,-0.91229 -7.59375,-2.46875 l -3.34375,1.59375 6.84375,14.15625 30,-14.46875 -6.8125,-14.15625 -6.21875,3 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" /> <path id="rect6068-7" d="m 296.78003,510.10489 c 5.18437,4.50697 5.94732,12.34021 1.62344,17.76227 -1.7712,2.22105 -4.11932,3.70233 -6.66474,4.39784 l -0.83873,3.60795 15.33484,3.4755 7.39233,-32.47612 -15.31536,-3.49993 -1.53178,6.73249 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" inkscape:connector-curvature="0" /> <path id="rect6068-0" d="m 301.92455,576.84686 c 6.75608,-1.2433 13.35607,3.04409 14.89936,9.80523 0.63219,2.76958 0.32629,5.52897 -0.71696,7.95272 l 2.29788,2.90524 12.27831,-9.82241 -20.78201,-26.02792 -12.28526,9.79193 4.30868,5.39521 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" inkscape:connector-curvature="0" /> <path id="rect6068-7-2" d="m 357.3147,614.41356 c 3.24024,-6.05733 10.70728,-8.54432 16.9556,-5.53547 2.55951,1.23253 4.52618,3.19214 5.7707,5.51895 l 3.70413,0.0148 -0.0242,-15.72374 -33.30682,0.0201 -0.004,15.71019 6.90455,-0.005 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" inkscape:connector-curvature="0" /> <path id="rect6068-7-4" d="m 421.16295,594.31034 c -2.71561,-6.30999 -0.004,-13.69859 6.24372,-16.70778 2.55944,-1.23266 5.31772,-1.54849 7.91284,-1.07077 l 2.32104,-2.8868 -12.30848,-9.78457 -20.75055,26.053 12.28029,9.79818 4.30097,-5.40135 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" inkscape:connector-curvature="0" /> <path id="rect6068-0-6" d="m 445.54357,531.88642 c -6.6265,-1.81104 -10.71254,-8.53772 -9.16986,-15.2986 0.63203,-2.76962 2.10483,-5.12305 4.09636,-6.85416 l -0.80987,-3.61453 -15.32408,3.52269 7.43155,32.46716 15.31712,-3.49217 -1.54139,-6.73031 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" inkscape:connector-curvature="0" /> <path id="rect6068-7-2-8" d="m 411.93706,474.00786 c -5.54745,4.05169 -13.3542,3.05231 -17.67827,-2.36908 -1.77132,-2.22096 -2.69304,-4.83979 -2.80482,-7.47615 l -3.33091,-1.62042 -6.80016,14.17724 30.01745,14.43252 6.81969,-14.15281 -6.22303,-2.99113 z" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" inkscape:connector-curvature="0" /> </ins> </g> </g> <del> <g id="g6152"> <path style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 345.7441,464.40625 c -0.29134,6.86335 -5.93996,12.34375 -12.875,12.34375 -2.84081,0 -5.46294,-0.91229 -7.59375,-2.46875 l -3.34375,1.59375 6.84375,14.15625 30,-14.46875 -6.8125,-14.15625 -6.21875,3 z" id="rect6068" inkscape:connector-curvature="0" /> <path inkscape:connector-curvature="0" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 296.78003,510.10489 c 5.18437,4.50697 5.94732,12.34021 1.62344,17.76227 -1.7712,2.22105 -4.11932,3.70233 -6.66474,4.39784 l -0.83873,3.60795 15.33484,3.4755 7.39233,-32.47612 -15.31536,-3.49993 -1.53178,6.73249 z" id="rect6068-7" /> <path inkscape:connector-curvature="0" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 301.92455,576.84686 c 6.75608,-1.2433 13.35607,3.04409 14.89936,9.80523 0.63219,2.76958 0.32629,5.52897 -0.71696,7.95272 l 2.29788,2.90524 12.27831,-9.82241 -20.78201,-26.02792 -12.28526,9.79193 4.30868,5.39521 z" id="rect6068-0" /> <path inkscape:connector-curvature="0" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 357.3147,614.41356 c 3.24024,-6.05733 10.70728,-8.54432 16.9556,-5.53547 2.55951,1.23253 4.52618,3.19214 5.7707,5.51895 l 3.70413,0.0148 -0.0242,-15.72374 -33.30682,0.0201 -0.004,15.71019 6.90455,-0.005 z" id="rect6068-7-2" /> <path inkscape:connector-curvature="0" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 421.16295,594.31034 c -2.71561,-6.30999 -0.004,-13.69859 6.24372,-16.70778 2.55944,-1.23266 5.31772,-1.54849 7.91284,-1.07077 l 2.32104,-2.8868 -12.30848,-9.78457 -20.75055,26.053 12.28029,9.79818 4.30097,-5.40135 z" id="rect6068-7-4" /> <path inkscape:connector-curvature="0" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 445.54357,531.88642 c -6.6265,-1.81104 -10.71254,-8.53772 -9.16986,-15.2986 0.63203,-2.76962 2.10483,-5.12305 4.09636,-6.85416 l -0.80987,-3.61453 -15.32408,3.52269 7.43155,32.46716 15.31712,-3.49217 -1.54139,-6.73031 z" id="rect6068-0-6" /> <path inkscape:connector-curvature="0" style="fill:#ffffff;fill-opacity:1;stroke:#ffffff;stroke-width:0;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" d="m 411.93706,474.00786 c -5.54745,4.05169 -13.3542,3.05231 -17.67827,-2.36908 -1.77132,-2.22096 -2.69304,-4.83979 -2.80482,-7.47615 l -3.33091,-1.62042 -6.80016,14.17724 30.01745,14.43252 6.81969,-14.15281 -6.22303,-2.99113 z" id="rect6068-7-2-8" /> </g> </del> </g> </g> </svg>

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --AWS EBS CSI driver returns topology segments It should merge the topology segments Enabling , and then describe the I have found in the docs which indicates that the duplication is by intention Also I found this and pull
/assign
/sig storage /tirage accepted /assign
It's caused by the migration from CSI to in-tree PV. AWS EBS CSI driver provides PV with: CSI translation changes -and we end up with two same items in the array. It makes sense to squash these duplicates.
cc what do you think?
Why is aws csi driver provides PV with Seems to me this is not needed.
I believe aws csi driver recently changed to using kubernetes topology labels instead of vendor-specific topology labels. Note that the elements of a matchExpression are together. I'm not sure if that's the correct approach if the intention is to eventually get rid of the vendor-specific topology label. cc
I think the topology label is defined by the csi driver? Is it okay to get rid of the vendor specific label?
The intention was to put both labels on all nodes to maintain backwards compatibility with existing PVs. but provisioned PVs should only have one or the other, putting both labels on the PV is a bug: Now that we've released the bug and some PVs have both labels, it will be hard to remove the vendor specific label.
thanks for your follow up. After the provisioned PV is assigned to a node, removing the duplicate label would still be an issue ?
It depends on which label is going to be removed. I'd suggest to remove the in-tree one (), because we can be sure that nodes have the other label from registering the CSI driver.
+1
/triage accepted /assign
fixed by Closing


Question: // Check iptables chain and rules epIpt := netlinktest.ExpectedIptablesChain{ string(kubeServicesChain): {{ <ins> JumpChain: string(KubeMarkMasqChain), MatchSet: kubeClusterIPSet, }, { </ins> JumpChain: string(KubeNodePortChain), MatchSet: "", <ins> }, { JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet, </ins> }}, string(KubeNodePortChain): {{ JumpChain: "RETURN", MatchSet: kubeNodePortLocalSetTCP, }, { JumpChain: string(KubeMarkMasqChain), MatchSet: kubeNodePortSetTCP, <ins> }, { JumpChain: "ACCEPT", MatchSet: kubeHealthCheckNodePortSet, </ins> }}, } checkIptables(t, ipt, epIpt)

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --ipvs mode didn't preserve client IP for NodePort Service with protocol SCTP and ExternalTrafficPolicy Local. iptables mode works fine. ipvs mode with TCP and UDP protocol works fine. Client IP for NodePort Service should be preseved when ExternalTrafficPolicy is Local regardless of the mode and the protocol. kube-proxy mode to ipvs a NodePort service with protocol SCTP and ExternalTrafficPolicy Local the NodePort service, the client IP was not preserved This is caused by wrong order of iptables rules that match kubeNodePortLocalSetSCTP and kubeNodePortSetSCTP: Kubernetes version (use ): v1.22.0 Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig network /assign
/triage accepted


Question: if len(line) == 0 { continue } <del> parts := strings.Split(line, " ") </del> // expected tc line: // class htb 1:1 root prio 0 rate 1000Kbit ceil 1000Kbit burst 1600b cburst 1600b <del> if len(parts) != 14 { return -1, fmt.Errorf("unexpected output from tc: %s (%v)", scanner.Text(), parts) </del> <ins> matches := classShowMatcher.FindStringSubmatch(line) if len(matches) != 2 { return -1, fmt.Errorf("unexpected output from tc: %s (%v)", scanner.Text(), matches) </ins> } <del> classes.Insert(parts[2]) </del> <ins> classes.Insert(matches[1]) </ins> } // Make sure it doesn't go forever

What happened: I tried to set up bandwidth limiting by booting a cluster with , which by default sets up kubenet. I used the annotations xM xM and then used to test the ingress and egress bandwidths, with no effect. Does anyone know if this feature is still supported in kubenet? And if so what are the steps I need to take to boot a cluster that can use pod annotations to do tc shaping. What you expected to happen: Per traffic shaping should be implemented on Kubenet as of 4/12/2018. Anything else we need to know?: There are a lot of issues regarding traffic shaping not working. None have a concrete answer of how to get it to work on kubenet: Relevant: I believe this is where it was first implemented. Environment: Kubernetes version (use ): 1.11.8 Cloud provider or hardware configuration: AWS CC:
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
cc
/triage unresolved I am a bot run by
can we put this on sig-network agenda? the discussion item would be: what is the current status of this feature (alpha, beta, GA?) what should we do next with this API?
I have this to the agenda, will grab some info.
AFAIK traffic shaping is still supported in kubenet. Does a pod with these annotations come up successfully but has no traffic shaping applied? Or do things fail earlier? If it comes up successfully, could you run ?
On GKE we do not officially support this, our default image even don't have tc installed. I tried with ubuntu, the kubenet is failed to parse the tc output. kubenet expect. // filter parent 1: protocol ip pref 1 u32 fh 800::800 order 2048 key ht 800 bkt 0 flowid 1:1 tc return.// filter parent 1: protocol ip pref 1 u32 chain 0 fh 800::800 order 2048 key ht 800 bkt 0 flowid 1:1 notinhw
If this issue has been triaged, please comment .

Meta: /athenabot mark-triage-reminder I am a bot run by
Depending on which CNI plugin you use you need to install the bandwidth plugin, see:
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
since it is in progress /priority critical-urgent /milestone v1.17
/lifecycle frozen
/assign
Bug triage for release 1.17 here. Since this issue has been tagged for the milestone, we would like to check it's status. Code freeze is on Nov. 14. Will this issue be resolved before then? If not, we should move it out of the milestone.
The fix is still pending in review.
Will it be merged by this Thursday for v1.17 code freeze?
/milestone v1.18 (v1.17 code freeze)
matching assigned priority on PR
Hello I'm one of the Bug Triage shadows for the 1.18 release. Since this issue is tagged for 1.18, I'd like to check its status. Is this issue intended for milestone 1.18? Friendly reminder: Code freeze is scheduled for March 5th.
Hello Checking in again. Do you think this will make it in for 1.18 release?
Hey Checking in again as this issue is marked as . Are you close to completing this for v1.18 release? Another reminder that code freeze is just around the corner! (March 5th).
Friendly reminder that code freeze is in 2 days and this issue is marked as
The fix got approved by ,should be able to catch the freeze.


Question: if err := runtime.Convert_runtime_RawExtension_To_runtime_Object(&curr.Extension, &newExtension, s); err != nil { return err } <ins> if *out == nil { *out = make(map[string]runtime.Object) } </ins> if (*out)[curr.Name] == nil { (*out)[curr.Name] = newExtension } else {

What happened: I have a test case: This behaves different with different client-go versions. With , the test passes. This is using in with in With , this test panics. This is using in with in Panic logs: Basically it its creating an object to decode into, and this object has some maps. These are set to nil, so when we try to populate them we get a panic. What you expected to happen: Behavior remains the same as previous release, or at least have some alternative that works here? How to reproduce it (as minimally and precisely as possible): I pasted the test above that reproduces this. This can be reproduced with a project with just that file, and a file with only client-go. Using go 1.13
/sig api-machinery
/assign


Question: // Block Volume case // Create volumeSpec from mount path <del> klog.V(5).Infof("Starting operationExecutor.ReconstructVolume") </del> <ins> klog.V(5).Infof("Starting operationExecutor.ReconstructVolume for block volume on pod %q", podName) </ins> // volumePath contains volumeName on the path. In the case of block volume, {volumeName} is symbolic link // corresponding to raw block device.

When a pod is marked as deleted while kubelet is down / being restarted, newly started kubelet does not clean up CSI filesystem volumes of the pod. Newly started kubelet tries to reconstruct the volume using CSI's function. This part looks working, CSI volume plugin loads its json file. But then VolumeManager checks if the volume is still mounted in directory. There are two issues: does not require volumes to be presented as mounts. They can be just directories with files on them. This will be case of the most of in-line volumes. Kubelet checks the right directory given by
/sig storage related to cc
Maybe a stupid idea... Reconciler sync has found a directory in . Reconciler has no way how to check that CSI volume (or any other volume) is really mounted / published there. Depending on DSW: The volume is in DSW -from comment here it seems that the SetUp (=NodePublish) will be called on the volume: The volume is not in DSW and it will be to ASW, so it's unmounted / unpublished in the next sync: In both cases, we can expect that the volume plugin / CSI driver is idempotent and SetUp() won't do anything if the volume is already set up / TearDown() will not do anything if the volume has been torn down already. Are our volume plugins really idempotent? IMO they are. We perhaps don't need to check for presence of mount point anywhere, just presence of directory should be enough to reconstruct the volume and set it up / tear it down. Adding to the loop.
Filed , PTAL
I'm curious how the reconstruction e2e tests passed
https://k8s-*force
They start kubelet with a pod already deleted. If the pod still exists, volume reconstruction can see it in DSW and does nothing when fails: [Reconstruction fails because it checks for a mount in a wrong directory.] In this case, kubelet waits for the volume to appear in VolumesInUse before calling . The pod is deleted before that and kubelet then just deletes DSW entry and the volume is never cleaned up.
They pass because their container does not handle SIGTERM and it takes 30 seconds to kill them. Kubelet has enough time to the volume during reconciliation.

It turns out that I tested with broken version of and reconstruction is broken only for ephemeral volumes that don't mount.
So the conclusion is that we only need to fix for ephemeral volumes? Thanks!
Yes, sorry again about the noise.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale cc
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
/remove-lifecycle rotten
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
/remove-lifecycle rotten
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or .

/lifecycle stale
/remove-lifecycle rotten
/lifecycle frozen
From this issue surfaced through e2e tests using the hostPath CSI driver. The cause is (which is used in for reconstruction) - it doesn't play well with volumes that bind mount from a directory instead of a device.
To work around this bug here, volume life cycle checking was disabled for all tests, not just the subpath test: This check is useful and should be enabled again.
Hey everyone! I arrived at this conversation by investigating on issue (seems to me like a duplicate of ). I'm wondering if there's something I can help with? I'm still a bit new to the codebase, though. I was able to reproduce the problem by using the following ginkgo focus command in the e2e framework: And do see that if we manually add the --check-volume-lifecycle=false to the statefulset/csi-hostpathplugin tests will pass correctly. I also see that during the process, something in the cleanup function doesn't correctly delete the pv's after the tests are done, so they remain alive if we execute a kubectl get pv command.
I think I found a bit of more info on the issue probably could be of some help. I think that the volume is not being correclty marked as unmounted inside in GenerateUnmountVolumeFunc. I'm not sure why but I'll continue to investigate on it. I think this error is coming from DeletePodFromVolume in
I a couple of extra debug lines to that lines in the file, and can see that when DeletePodFromVolume tries to see if the volume exists, it doesn't find it in the asw.attachedVolumes struct, like so: Probably we should search for the name of the attached volume differently for ephemeral volumes?
/assign
There are two problems: 1) The IsLikelyNotMountPoint() call that , which is known not to work for bind mounts. The mountpoint never gets found, so volume reconstruction fails with "... is not mounted", the PVC never never gets to the list of attached volumes, and that's why we end up hitting this error: 2) CSI inline volumes hit a different error (in addition to the first problem). The call path looks like this (starting in ): Inline volumes do not have a PV spec, so it fails . We should be calling instead.

There is an ongoing that changes to utilize syscall to detect mount point by using . By using , bind mount will be properly detected fast, but it requires kernel version is v5.6 or later. So, we would be able to also utilize in . Then, the issue of bind mount can be resolved at least for kernel v5.6+, and we will be able to focus on how we solve this issue for old kernels.


Question: err = WaitForPodSuccessInNamespace(client, injectPod.Name, injectPod.Namespace) Expect(err).NotTo(HaveOccurred()) } <ins> func CreateGCEVolume() (*v1.PersistentVolumeSource, string) { diskName, err := CreatePDWithRetry() ExpectNoError(err) return &v1.PersistentVolumeSource{ GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ PDName:   diskName, FSType:   "ext3", ReadOnly: false, }, }, diskName } </ins>

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --We should have an e2e to test issue . The test needs to delete a PVC while a pod is still using it, and then restart the controller manager. /kind bug /sig storage
CC
/assign unofficially so I can find this later.


Question: if kl.resolverConfig == "" { hostDNS = []string{"127.0.0.1"} hostSearch = []string{"."} <ins> } else { hostSearch = kl.formDNSSearchForDNSDefault(hostSearch, pod) </ins> } return hostDNS, hostSearch, nil }

Kubernetes version v1.3.0 Kubelet concatenates host and cluster domain entries for my pod, resulting in an invalid config (too many entries with duplicates). The max is only 6 search domains. Any more and DNS resolution breaks when trying to leverage search domains. The example below has only cluster DNS entries, but illustrates the issue: Host search nameserver 10.233.0.2 nameserver 10.90.0.1 options timeout:2 options attempts:2 Container kubectl run -i --tty busybox --image=busybox -- sh (skip waiting text) / # cat search nameserver 10.233.0.2 options attempts:2 options ndots:5 I should be enable cluster DNS config in my host DNS settings and still have functional pod DNS. Kubelet should be intelligent enough to filter out duplicate search domain entries (or drop search domains beyond the 6th entry)
I think some sane checks needs to be too. libc limits search field to 6 records and 255(?) characters. This limitation should be checked before creation.
Hey guys, I'm working on it.
Dup of


Question: if len(s.Username) == 0 { return fmt.Errorf("username must be specified") } <del> if len(s.Email) == 0 { return fmt.Errorf("email must be specified") } </del> if len(s.Password) == 0 { return fmt.Errorf("password must be specified") }

<!-- Thanks for filing an issue! Before hitting the button, please answer these questions.--Is this a request for help? (If yes, you should use our troubleshooting guide and community support channels, see ): What keywords did you search in Kubernetes issues before filing this one? (If you have found any duplicates, you should instead reply there.): Is this a BUG REPORT or FEATURE REQUEST? (choose one): <!-- If this is a BUG REPORT, please: Fill in as much of the template below as you can. If you leave out information, we can't help you as well. If this is a FEATURE REQUEST, please: Describe in detail the feature/behavior/change you'd like to see. In both cases, be ready for followup questions, and please respond in a timely manner. If we can't reproduce a bug or think a feature already exists, we might close your issue. If we're wrong, PLEASE feel free to reopen it and explain why. --Kubernetes version (use ): Environment: Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others: What happened: When I create a docker registry secret, the is also a MUST, I think we should make this parameter as optional as there is no need always put when creating a docker registry secret. What you expected to happen: How to reproduce it (as minimally and precisely as possible): Anything else we need to know:
Agree it should not be required
Did this change get reverted? I do not see as optional on stable.
I feel that this code should be changed because "--docker-email flag" is no longer required.
error: flag docker-email is required. On kubenetes 1.10.3, the above error message still exists
The field is still required. This is very frustrating as does not require an email but Kubernetes does? Is there a good reason for this?
Why is this closed? As far as I can see it is still required?


Question: func newCacheWatcher(copier runtime.ObjectCopier, resourceVersion uint64, chanSize int, initEvents []*watchCacheEvent, filter watchFilterFunc, forget func(bool)) *cacheWatcher { watcher := &cacheWatcher{ copier:  copier, <del> input:   make(chan *watchCacheEvent, chanSize), </del> <ins> input:   make(chan watchCacheEvent, chanSize), </ins> result:  make(chan watch.Event, chanSize), done:    make(chan struct{}), filter:  filter,

Our scalability tests started failing on Thu last week: https://k8s- [250 run passed, 251 failed and since then everything failed] I started looking into it and it seems that something happened to how we are handling watches (i.e. I'm seeing a huge number of watches failing with "err too old" errors (which I didn't see before).
As an example in run 258 of kubemark-scale, I'm seeing this log in the test (after 5 minutes of running test); This is extremely strange and unexpected. I looked into apiserver logs, and there is expected number of POST and PUT operations on pods (and no DELETE and PATCH operations until then). Until then, there were alltogethere ~60.000 operations on pods. However, in 5000-node cluster, the size of watchcache of pods is supposed to be 250.000, which means that at that moment we should have all the history from the beginning in watch cache. And still we are observing those errors. Something must have changed around that recently...
As an example, logs from apiserver from a given node: This means something is really wrong: the first watch started at 45, then after it finished, we tried starting watch from rv= and it failed with "rv too old" the list, returned rv= from which we started the next watch. If the watch cache size for pods would be 250.000 (50*cluster size), there there is no way that with current rv=, the rv= would be outside history window...
I had a hypothesis that it might be the problem of breaking watch to etcd. But in that case there should be a warning "Terminating all watchers" log in apiserver, and there is no such log. It also seems that the size of cache is being set correctly...
So I tried running large kubemark locally and it seems that the problem is that the size of cache is too small. I few more logs, and here is what I'm seeing: So it seems that we are correctly initializing cachesizes, but then watch cache capacity is 100 (which explains why we are seeing all those "rv too old" errors. I will try to debug why the capacity is so small.
OK - I found it. contains the fix.


Question: return true, pod, err } if errs := validation.ValidatePod(newPod); len(errs) > 0 { <del> err = fmt.Errorf("invalid pod: %v", errs) return true, pod, err </del> <ins> return true, pod, fmt.Errorf("invalid pod: %v", errs) </ins> } v1Pod := &v1.Pod{} if err := k8s_api_v1.Convert_api_Pod_To_v1_Pod(newPod, v1Pod, nil); err != nil { <ins> glog.Errorf("Pod %q failed to convert to v1", newPod.Name) </ins> return true, nil, err } return true, v1Pod, nil

To read a pod manifest file from disk (or from the http source), kubelet execute the following code to Decode a on-disk v1 pod manifest to an internal pod object (the defaults are applied in this process). Validate the internal pod. Convert the internal pod back to a v1 pod. The downstream pod cache (which gets v1 pods from different sources) performs similar operations: Convert the v1 pod to an internal pod. Validate the internal pod. One of two conversion/validation should be removed, and some of them should be simplified.
/cc for suggestions on defaulting and conversion.
If no one is working on this, I will take it.


Question: <del> /var/lib/docker/containers/*/*-json.log { rotate 5 copytruncate missingok notifempty compress maxsize 10M daily create 0644 root root } </del>

We have rotation of logs from docker containers, but not for logs from docker Without rotation, the logs may fill the root partition entirely.
I can add rotation (5 files, 100MB each) if you decide it goes for v1.0 (for me it's a blocker). I'm not sure which platforms are affected. For sure GCE. cc
Sure, send a PR and I'm sure we can merge it.


Question: return err } <ins> func lookEmitSignalCommand() error { _, err := exec.LookPath("dbus-send") return err } </ins> func getNodeReadyStatus(f *framework.Framework) bool { nodeList, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) framework.ExpectNoError(err)

Failing on cri-o serial job. Looks like the dbus command isn't available for graceful shutdown tests that aren't running on Ubuntu. /assign /kind failing-test <!-- If this is a flake, please add: /kind flake --/sig node
/triage accepted /priority important-soon
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
hmm, is this still issue? The testgrid job is still failing even after the fix appears to be merged? !
Please take a look when you get a chance, the error still seems to be that dbus-send is unavailable... /cc
oops... I forgot to add this to the other


Question: kindString = strings.ToLower(groupKind.Kind) } <ins> if gracePeriod == 0 { operation = "force deleted" } </ins> if shortOutput { // -o name: prints resource/name fmt.Fprintf(out, "%s/%sn", kindString, info.Name)

IMO it's surprising that does not force delete a resource. only takes effect when . The description of reads this: We should make it more clear what does and even print warnings when is specified without .
Changing the meaning of force to actually break cluster consistency be default could lead to data loss, which is why we haven’t changed it so far. agree we should print more info and clarfiy


Question: AfterEach(func() { cleanupLocalVolume(config, testVol) }) <del> It("should be able to mount volume, write from pod1, and read from pod2 using one-command containers", func() { By("Creating pod1 to write to the PV") writeCmd, readCmd := createWriteAndReadCmds(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/) writeThenReadCmd := fmt.Sprintf("%s;%s", writeCmd, readCmd) podSpec1 := makeLocalPod(config, testVol, writeThenReadCmd) f.TestContainerOutput("pod writes to PV", podSpec1, 0, []string{testVol.hostDir}) By("Creating pod2 to read from the PV") podSpec2 := makeLocalPod(config, testVol, readCmd) f.TestContainerOutput("pod reads PV", podSpec2, 0, []string{testVol.hostDir}) }) </del> }) LocalVolumeTypes := []LocalVolumeType{DirectoryLocalVolumeType, TmpfsLocalVolumeType}

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug /assign What happened: Looks like the event messaging has changed


Question: if err != nil { klog.ErrorS(err, "Failed to get node ip address matching nodeport cidrs, services with nodeport may not work as intended", "CIDRs", proxier.nodePortAddresses) } <ins> // nodeAddresses may contain dual-stack zero-CIDRs if proxier.nodePortAddresses is empty. // Ensure nodeAddresses only contains the addresses for this proxier's IP family. isIPv6 := proxier.iptables.IsIPv6() for addr := range nodeAddresses { if utilproxy.IsZeroCIDR(addr) && isIPv6 == netutils.IsIPv6CIDRString(addr) { // if any of the addresses is zero cidr of this IP family, non-zero IPs can be excluded. nodeAddresses = sets.NewString(addr) break } } </ins> // Build rules for each service. for svcName, svc := range proxier.serviceMap {

I've been following the official docs for kubeadm and how to set up a new cluster. The cluster is up and running but I noticed warning events when creating node port services: However, the node port does work (I see the output when accessing node ip:nodeport). Might it be related to something else than Kubernetes? There is no such warning event. Set up a fresh cluster with kubeadm and then create a service of nodeport type. No response <details</details<detailsHetzner cloud </details<details</details<detailscloud-init </details<detailscontainerd </details<detailscni=cilium csi=hetzner </details
There are no sig labels on this issue. Please add an appropriate label by using one of the following commands: - Please see the for a listing of the SIGs, working groups, and committees available. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
check with netstat or ss if there is a process listening on that port
no, I specified exact random port e.g. and checked it before with ss - it was not being used. Then I created the service, the warning occured, and after that the port was taken:
kube-proxy binds the NodePort to warn users that they may be shadowing a process, but it logs the errors if it can not bind the port
I met the same error and I confirmed the node port was not used by any other process. After looking into the code, I think this is a bug in kube-proxy that it would try to open a node port twice with same (empty) IP and port when is empty. contains and in below code.: I have created to fix it.
Awesome find. Did you figure out a work around?

I am encountering this issue with this version of the image:
The error looked scaring but the nodePort should be working, as it was just caused by a duplicate port opening attempt. v1.23.5 and later releases have the fix.


Question: Testname: Networking, intra pod http, from node Description: Create a hostexec pod that is capable of curl to netcat commands. Create a test Pod that will act as a webserver front end exposing ports 8080 for tcp and 8081 for udp. The netserver service proxies are created on specified number of nodes. The kubectl exec on the webserver container MUST reach a http port on the each of service proxy endpoints in the cluster using a http post(protocol=tcp)  and the request MUST be successful. Container will execute curl command to reach the service port within specified max retry limit and MUST result in reporting unique hostnames. <ins> This test is marked LinuxOnly it breaks when using Overlay networking with Windows. </ins> */ <del> framework.ConformanceIt("should function for node-pod communication: http [NodeConformance]", func() { </del> <ins> framework.ConformanceIt("should function for node-pod communication: http [LinuxOnly] [NodeConformance]", func() { </ins> config := e2enetwork.NewCoreNetworkingTestConfig(f, true) for _, endpointPod := range config.EndpointPods { err := config.DialFromNode("http", endpointPod.Status.PodIP, e2enetwork.EndpointHTTPPort, config.MaxTries, 0, sets.NewString(endpointPod.Name))

ci-kubernetes-e2e-capz-master-containerd-windows should function for node-pod communication: http [NodeConformance] should function for node-pod communication: udp [NodeConformance] PR merged on April 4th 2022 Discussed at sig-windows triage meeting as well as regular meeting: This test is passing for flannel overlay: and aks-engine: We plan temporary disable this on the CAPZ test suite while we diagnose what is missing. /sig windows /sig networking /cc
The label(s) cannot be applied, because the repository doesn't have them. <detailsIn response to : ### Which jobs are failing? should function for node-pod communication: udp [NodeConformance] ### Since when has it been failing? /sig networking Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig network Did some looking into this and the tests is not deploying a hostprocess pod for Windows: Following the test flow Creates test pods: Then the hostnetwork is set to true but the windows Hostprocess fields are not set:
This pod being tested has which seems to be causing issues with calico, if I remove that then it works (as the other tests that do pod-pod demonstrate). Once I switched this test to use a hostprocess this test works for on the same host when using Windows overlay. Overlay has this limitation that we have seen before: fail (according to the , it is not mandatory for kubelet to be able to access pods on a different node).
My thoughts are we should disable these two tests for 1.24 because they are not testing what they should be and it requires some tough changes to the framework to make it work as is. These tests also doesn't make sense for . A example of the changes need to make this partially work on l2bridge but not overlay:


Question: } } return nil <del> }, 2*time.Minute, time.Second*4).Should(gomega.BeNil()) </del> <ins> }, 30*time.Second, time.Second*4).Should(gomega.BeNil()) </ins> // TODO: Test the oom-score-adj logic for burstable more accurately. })

Test: [sig-node] Restart [Serial] [Slow] [Disruptive] Kubelet should correctly account for terminated pods after restart I've also seen on /kind flake /sig node Looks like occasionally we have a second 50m CPU pod hanging around on the node, blocking enough pods getting scheduled for the test: I'm a bit puzzled because no pods are listed as currently running when the node dumps debug info. Could be a bug?
another case [sig-node] Container Manager Misc [Serial] Validate OOM score adjustments [NodeFeature:OOMScoreAdj] affected the Restart test case, see the kubelet log, the kubelet-container-manager guaranteede9d81584 pod was received almost simultaneously with the restart-test pod
Since at 0316 21:25:08, the container of pod guaranteede9d81584 has just been creating, but the test case [sig-node] Container Manager Misc [Serial] has already finished at 0316 21:25:06, so the test case check condition may be not strict
/triage accepted /assign
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle rotten [1]:
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs.

This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Reopen this issue or PR with Mark this issue or PR as fresh with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /close [1]:
Closing this issue. <detailsIn response to : The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closedReopen this issue or PR with Mark this issue or PR as fresh with Offer to help out with [Issue Triage][1] Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. <del> KUBE_COVER=" " KUBE_RACE=" " "${KUBE_ROOT}/hack/test-go.sh" "" -test.run="^X" -benchtime=1s -bench=. -benchmem </del> <ins> KUBE_COVER="" KUBE_RACE=" " "${KUBE_ROOT}/hack/test-go.sh" -- -test.run="^X" -benchtime=1s -bench=. -benchmem </ins>

Our test- script invokes 'go test' serially for each pkg - currently taking 2 minutes on my laptop. Running a single 'go test' for all pkgs cuts the time down to 45 seconds. Even running with -race is only 1:15. As far as I can tell, the only time we HAVE to run serially is when coverage is on. Does anyone have time to fix this?
I'd created on Friday to fix this, though it's not yet been reviewed.
I cobbled together but I'll review yours since it is probably better :) , Jeff Grafton wrote:


Question: By(fmt.Sprintf("dialing(udp) %v (node) --> %v:%v (nodeIP)", config.NodeIP, config.NodeIP, config.NodeUdpPort)) config.DialFromNode("udp", config.NodeIP, config.NodeUdpPort, config.MaxTries, config.MaxTries, sets.NewString()) }) <del> // TODO: Test sessionAffinity #31712 </del> <ins> It("should function for client IP based session affinity: http", func() { config := framework.NewNetworkingTestConfig(f) By(fmt.Sprintf("dialing(http) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.ClusterIP, framework.ClusterHttpPort)) updateSessionAffinity := func(svc *v1.Service) { svc.Spec.SessionAffinity = v1.ServiceAffinityClientIP } framework.UpdateService(f.ClientSet, config.NodePortService.Namespace, config.NodePortService.Name, updateSessionAffinity) firstEndpoints, err := config.GetEndpointsFromTestContainer("http", config.ClusterIP, framework.ClusterHttpPort, config.MaxTries, 0) if err != nil { framework.Failf("Unable to get endpoints from test containers: %v", err) } for i := 0; i < framework.SessionAffinityChecks; i++ { eps, err := config.GetEndpointsFromTestContainer("http", config.ClusterIP, framework.ClusterHttpPort, config.MaxTries, 0) if err != nil { framework.Failf("Unable to get endpoints from test containers: %v", err) } if !eps.Equal(firstEndpoints) { framework.Failf("Expect endpoints: %v, got: %v", firstEndpoints, eps) } } }) It("should function for client IP based session affinity: udp", func() { startTime := time.Now() config := framework.NewNetworkingTestConfig(f) By(fmt.Sprintf("dialing(udp) %v --> %v:%v (config.clusterIP)", config.TestContainerPod.Name, config.ClusterIP, framework.ClusterHttpPort)) updateSessionAffinity := func(svc *v1.Service) { svc.Spec.SessionAffinity = v1.ServiceAffinityClientIP } framework.UpdateService(f.ClientSet, config.NodePortService.Namespace, config.NodePortService.Name, updateSessionAffinity) firstEndpoints, err := config.GetEndpointsFromTestContainer("udp", config.ClusterIP, framework.ClusterUdpPort, config.MaxTries, 0) if err != nil { framework.Failf("Unable to get endpoints from test containers: %v", err) } for i := 0; i < framework.SessionAffinityChecks; i++ { eps, err := config.GetEndpointsFromTestContainer("http", config.ClusterIP, framework.ClusterUdpPort, config.MaxTries, 0) if err != nil { framework.Failf("Unable to get endpoints from test containers: %v", err) } if !eps.Equal(firstEndpoints) { framework.Failf("Expect endpoints: %v, got: %v", firstEndpoints, eps) } } framework.Failf("test session affinity, cost time: %v", time.Now().Sub(startTime)) }) </ins> }) })

Augment to turn on session affinity for a service, lookup the vip from a container/node and make sure we only hit a single endpoint with N ready/running endpoints.
This looks like it is still relevant can anyone confirm if we these? I still see a
/assign
Are you working on it? Could you please assign this issue to me and I am happy to fix it. Thanks a lot!
/assign I have it in my "backlog" behind NetworkPolicy stuff. Feel free to take it over if you have time.
GitHub didn't allow me to assign the following users: m1093782566. Note that only can be assigned. <detailsIn response to : /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Would appreciate :)
/assign


Question: } // TargetContainer <del> if len(o.TargetContainer) > 0 && len(o.CopyTo) > 0 { return fmt.Errorf("--target is incompatible with --copy-to. Use --share-processes instead.") </del> <ins> if len(o.TargetContainer) > 0 { if len(o.CopyTo) > 0 { return fmt.Errorf("--target is incompatible with --copy-to. Use --share-processes instead.") } if !o.Quiet { // If the runtime doesn't support container namespace targeting this will fail silently, which has caused // some confusion (ex: https://issues.k8s.io/98362), so print a warning. This can be removed when // EphemeralContainers are generally available. fmt.Fprintf(o.Out, "Targeting container %q. If you don't see processes from this container it may be because the container runtime doesn't support this feature.n", o.TargetContainer) // TODO(verb): Add a list of supported container runtimes to https://kubernetes.io/docs/concepts/workloads/pods/ephemeral-containers/ and then link here. } </ins> } // TTY

What happened: Cannot enter the process namespace using with target parameter. The nginx process does not appear, but a pause process instead. What you expected to happen: Enter the process namespace of a target container and see processes running inside it. How to reproduce it (as minimally and precisely as possible): a Pod. with target parameter and show processes. Anything else we need to know?: Environment: Kubernetes version (use ): 1.19.7 OS (e.g: ):Kernel (e.g. ): Install tools: CKE Network plugin and version (if this is a network-related bug): Coil v2.0.5 Others: Container Runtime: containerd 1.4.3
/sig cli
Add the parameter to try
I know I can see all processes including nginx and pause with , but why parameter doesn't work as expected on my environment? I thought it targets the process namespace of another container.
/assign Please take a look and accept if so.
/triage accepted
I suspect this is because containerd hasn't implemented This works for me with the builtin dockershim: I know that dockershim is slated for removal soon, so we should ask containerd to support this.
We should probably print a warning that requires support from the runtime
Opened to discuss this with the containerd authors.
Did you try this with ? I tried which didn't work for me with the message : Error: unknown flag: --share-processes See 'kubectl alpha debug --help' for usage. ( And this doesn't list --share-processes as an option ) this suggests me that kubectl alpha debug doesn't have --share-processes option enabled at least until kubectl version v1.18.4 Environment : Kubernetes version (use kubectl version): 1.18.4 Others: Container Runtime: containerd 1.3.3
containerd now has an open PR to fix this (containerd/containerd)
containerd/containerd has been resolved. containerd plans to support namespace targeting in the 1.5 release.
/close
Closing this issue. <detailsIn response to : /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: // service or not.  If this field is specified when creating a Service // which does not need it, creation will fail. This field will be wiped // when updating a Service to no longer need it (e.g. changing type). <ins> // This field cannot be updated once set. </ins> // +optional HealthCheckNodePort int32 `json:"healthCheckNodePort,omitempty" protobuf:"bytes,12,opt,name=healthCheckNodePort"`

We are currently using v1.19 and it seems that if you have a service with set to then changing the health check node port is not supported. One way to get around this is by changing the to and then changing it back to in addition to specifying the with the desired port number. This behavior is not very well documented and the below error message is not very useful: I wonder if we could also add another case below to release the current health check port and allow changing it to the user specified value given it's valid? It's possible that after creating a someone wants to change it to a user-provided and the current way to do it is only by toggling the . This feature with the documentation would be more straightforward way to make any port changes.
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig network
19 is EOL, last supported version is 1.22 It seems the message has changed since then, this area had important changes in latest releases that improved the API behavior However, the field still can not be updated once set, what you do switching the is wiping the value and reallocating it. It is a documentation problem then
IMO, the amount of effort it would take to make this mutable is not justified when there is a workaround. Convince me the workaround doesn't work? Overall HCNP is a buried implementation detail.
I can't speak much to the amount of effort, but I can speak a bit to the value: a user, I don't expect healthcheckNodePort to be immutable. I'm not sure if my mental model is too simplistic, but it feels like a part of the configuration I control, and I can change NodePorts, so why not the healthcheckNodePort. the work around isn't necessarily obvious. It also takes experimenting with to know that it'll even work. this in a production environment is not trivial.

If changing the healthcheckNodePort was supported then you could change the config to the state you want it to be, and your rollout orchestration tooling goes and applies it to your collection of k8s clusters. With the work around you now have to have a 2 step process. After step 1 of this process your service is using a traffic policy which might break your application (say you depend on client ip). So to avoid the downtime you'd have to build some reasonably off-pattern orchestration that doesn't just apply the configuration from the commit you used for your release, but instead two commits, or two configs, etc. Operationally this isn't actually that cheap to build, especially as this is likely to be a one-off operation and not something you do every release. Not exactly the same thing, but related to healthcheckNodePort, I've also found that specifying it explicitly vs not explicitly in the config, and switching between the two modes, has unexpected behavior as well. If you started out as having it specified, applying a config with it unspecified tries to allocate a new value, and complains that it can't change healthcheckNodePort for traffic policy local. Why not just use the already allocated value? Going from unallocated to specifying it explicitly at the same value seems to works fine.
The latter issue should be fixed in recent releases - what version are you seeing that on? I understand it would be more consistent to be mutable but there are real downstream implications to changing it, code complexity, test complexity, etc, and I don't understand why you want to change it in the first place. Humans should not be paying attention to that field. It should not even be in spec, but for historical reasons.
Same version as original post, v1.19. Glad to hear that issue is fixed in later versions! We're making steady progress updating, but not all caught up yet. I don't want to insist on this fix, as I said I'm not aware of the fix complexity and leave it up to you guys to balance that out, just wanted to provide the value perspective. As for not being exposed to users at all, I think we'd probably have been better off if we never knew about it.

Unfortunately our original configs had it explicitly set, eventually ran into conflicts in various k8s clusters and tried to transition to having it be unspecified, and thus ran into the above situation.
Yeah, there's really no good reason to set it, pretty much ever, and lots of reasons not to. I will cross-check that it is fixed if I get a moment tomorrow.
It has been fixed by you Tim
I thought so, I just wanted to triple check :)


Question: return err } } <ins> type TemplatePrinter struct { Template *template.Template } func (t *TemplatePrinter) Print(data []byte, w io.Writer) error { obj, err := api.Decode(data) if err != nil { return err } return t.PrintObj(obj, w) } func (t *TemplatePrinter) PrintObj(obj interface{}, w io.Writer) error { return t.Template.Execute(w, obj) } </ins>

To remove dependence on jq and generally make scripting much easier, we should make another output flag which takes a formatter. We can use go's template package, which is very flexible.


Question: changes.CreateSandbox = false return changes } <ins> // Get the containers to start, excluding the ones that succeeded if RestartPolicy is OnFailure. var containersToStart []int for idx, c := range pod.Spec.Containers { if pod.Spec.RestartPolicy == v1.RestartPolicyOnFailure && containerSucceeded(&c, podStatus) { continue } containersToStart = append(containersToStart, idx) } // We should not create a sandbox for a Pod if initialization is done and there is no container to start. if len(containersToStart) == 0 { _, _, done := findNextInitContainerToRun(pod, podStatus) if done { changes.CreateSandbox = false return changes } } </ins> if len(pod.Spec.InitContainers) != 0 { // Pod has init containers, return the first one. changes.NextInitContainerToStart = &pod.Spec.InitContainers[0] return changes } <del> // Start all containers by default but exclude the ones that succeeded if // RestartPolicy is OnFailure. for idx, c := range pod.Spec.Containers { if containerSucceeded(&c, podStatus) && pod.Spec.RestartPolicy == v1.RestartPolicyOnFailure { continue } changes.ContainersToStart = append(changes.ContainersToStart, idx) } </del> <ins> changes.ContainersToStart = containersToStart </ins> return changes }

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: For a Pod with , kubelet would attempt to create a new sandbox for it even after all container succeeded. Logs as below: This issue is similar to which fixed the case of pods whose RestartPolicy are Never. It caused unnecessary CRI and CNI calls, confusing logs and conflicts between the routine that creates the new sandbox and the routine that kills the Pod. It's likely also the root cause of because sometimes the routine that creates the second sandbox and the routine that kills unwanted pod run concurrently, then the netns would be gone when calling CNI. What you expected to happen: kubelet shouldn't create a new sandbox for pod with RestartPolicyOnFailure if all containers succeeded How to reproduce it (as minimally and precisely as possible): Create a pod like this: Anything else we need to know?: Environment: Kubernetes version (use ): v1.18.2 Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig node
Created which may fix this.
/assign