)]}'
{"id":"lttng-tools~10218","triplet_id":"lttng-tools~stable-2.13~I8dc1ee45dd00c85556dd70d34a3af4f3a4d4e7cb","project":"lttng-tools","branch":"stable-2.13","attention_set":{},"removed_from_attention_set":{"1000006":{"account":{"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},"last_update":"2023-06-27 18:10:43.000000000","reason":"Change was marked work in progress"},"1000033":{"account":{"_account_id":1000033,"name":"Erica Bugden","display_name":"Erica Bugden","email":"ebugden@efficios.com","username":"ebugden","avatars":[{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},"last_update":"2023-07-25 20:47:59.000000000","reason":"Change was marked work in progress"}},"hashtags":[],"change_id":"I8dc1ee45dd00c85556dd70d34a3af4f3a4d4e7cb","subject":"Fix: sessiond: bad fd used while rotating exiting app\u0027s buffers","status":"MERGED","created":"2023-06-06 21:37:23.000000000","updated":"2023-07-27 17:50:33.000000000","submitted":"2023-07-27 17:50:33.000000000","submitter":{"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},"total_comment_count":2,"unresolved_comment_count":0,"has_review_started":true,"submission_id":"10218","meta_rev_id":"dc4eba66aee43154f8949055fbab3b1d9ffbd493","_number":10218,"virtual_id_number":10218,"owner":{"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},"actions":{},"labels":{"Code-Review":{"all":[{"value":0,"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]}],"values":{"-2":"This shall not be merged","-1":"I would prefer this is not merged as is"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me, approved"},"description":"","default_value":0,"optional":true},"Verified":{"all":[{"value":0,"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]}],"values":{"-2":"Failure","-1":"Not built"," 0":"No score","+1":"Unstable","+2":"Success"},"description":"CI Build results","default_value":0,"optional":true},"CI-Build":{"all":[{"value":0,"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]}],"values":{" 0":"No score","+1":"Trigger a CI Build only","+2":"Trigger a CI Build with Fast Tests","+3":"Trigger a CI Build with Complete Tests"},"description":"Trigger CI builds","default_value":0,"optional":true},"Smoke-Build-Lvl1":{"all":[{"value":0,"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]}],"default_value":0,"optional":true},"Smoke-Build-Lvl2":{"all":[{"value":0,"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]}],"default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},{"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]}],"CC":[{"_account_id":1000033,"name":"Erica Bugden","display_name":"Erica Bugden","email":"ebugden@efficios.com","username":"ebugden","avatars":[{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2023-06-06 21:38:24.000000000","updated_by":{"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"reviewer":{"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"state":"CC"},{"updated":"2023-06-06 22:43:59.000000000","updated_by":{"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"reviewer":{"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"state":"REVIEWER"},{"updated":"2023-06-26 21:09:44.000000000","updated_by":{"_account_id":1000033,"name":"Erica Bugden","display_name":"Erica Bugden","email":"ebugden@efficios.com","username":"ebugden","avatars":[{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},"reviewer":{"_account_id":1000033,"name":"Erica Bugden","display_name":"Erica Bugden","email":"ebugden@efficios.com","username":"ebugden","avatars":[{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},"state":"CC"}],"messages":[{"id":"de8cac0d9f009584063a825d23880a62b6729d17","tag":"autogenerated:gerrit:newWipPatchSet","author":{"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},"date":"2023-06-06 21:37:23.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"8cde5e7bec7a19b6208fb9a8d3a9d566daeda9ab","author":{"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},"date":"2023-06-06 21:38:17.000000000","message":"Patch Set 1: CI-Build+1","accounts_in_message":[],"_revision_number":1},{"id":"ddb528c674f835973cfef8fa88e5900c699d0221","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2023-06-06 21:38:24.000000000","message":"Patch Set 1:\n\nBuild Started https://ci.lttng.org/job/dev_review_lttng-tools_stable-2.13_build/33/ (1/2)","accounts_in_message":[],"_revision_number":1},{"id":"0abf121329054515313d53e8117308849f29140d","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2023-06-06 21:38:24.000000000","message":"Patch Set 1:\n\nBuild Started https://ci.lttng.org/job/dev_review_lttng-tools_stable-2.13_rootbuild/32/ (2/2)","accounts_in_message":[],"_revision_number":1},{"id":"11c72cf23ffb628313e23180d2161caa4c1de168","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2023-06-06 22:43:59.000000000","message":"Patch Set 1: Verified+2\n\nBuild Successful \n\nhttps://ci.lttng.org/job/dev_review_lttng-tools_stable-2.13_rootbuild/32/ : SUCCESS\n\nhttps://ci.lttng.org/job/dev_review_lttng-tools_stable-2.13_build/33/ : SUCCESS","accounts_in_message":[],"_revision_number":1},{"id":"dcebf25040a31c9cff0987947250253541790f0a","author":{"_account_id":1000033,"name":"Erica Bugden","display_name":"Erica Bugden","email":"ebugden@efficios.com","username":"ebugden","avatars":[{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/c42f50f12968dff6041103eb3b1af3bd.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},"date":"2023-06-26 21:09:44.000000000","message":"Patch Set 1:\n\n(1 comment)","accounts_in_message":[],"_revision_number":1},{"id":"174c6c2f75e317e1ffbc6a2634c08b6e9984759a","tag":"autogenerated:gerrit:newWipPatchSet","author":{"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},"date":"2023-06-27 18:10:43.000000000","message":"Uploaded patch set 2: Commit message was updated.\n\nCopied Votes:\n* CI-Build+1 (copy condition: \"changekind:NO_CHANGE OR **changekind:NO_CODE_CHANGE**\")\n* Verified+2 (copy condition: \"changekind:NO_CHANGE OR **changekind:NO_CODE_CHANGE**\")\n","accounts_in_message":[],"_revision_number":2},{"id":"c550cc0128ccbc565e7339bc9e1d82f59e7c105f","author":{"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},"date":"2023-06-27 18:11:07.000000000","message":"Patch Set 2: CI-Build+1\n\n(1 comment)","accounts_in_message":[],"_revision_number":2},{"id":"2f13a8b5852559a86f06dd131870bce2bf288147","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2023-06-27 18:11:16.000000000","message":"Patch Set 2: -Verified\n\nBuild Started https://ci.lttng.org/job/dev_review_lttng-tools_stable-2.13_build/35/ (1/2)","accounts_in_message":[],"_revision_number":2},{"id":"2347e667d6277ec94493431f2d2c37b2383e5cd2","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2023-06-27 18:11:16.000000000","message":"Patch Set 2:\n\nBuild Started https://ci.lttng.org/job/dev_review_lttng-tools_stable-2.13_rootbuild/34/ (2/2)","accounts_in_message":[],"_revision_number":2},{"id":"c4b86dc062f59cc026dea580c72f0a63fe65604f","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2023-06-27 19:17:15.000000000","message":"Patch Set 2: Verified+2\n\nBuild Successful \n\nhttps://ci.lttng.org/job/dev_review_lttng-tools_stable-2.13_build/35/ : SUCCESS\n\nhttps://ci.lttng.org/job/dev_review_lttng-tools_stable-2.13_rootbuild/34/ : SUCCESS","accounts_in_message":[],"_revision_number":2},{"id":"38d679be5d72b137f5f09ef41043eb2045ff9f85","tag":"autogenerated:gerrit:newWipPatchSet","author":{"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},"date":"2023-07-25 20:47:59.000000000","message":"Uploaded patch set 3.\n\nOutdated Votes:\n* CI-Build+1 (copy condition: \"changekind:NO_CHANGE OR changekind:NO_CODE_CHANGE\")\n* Verified+2 (copy condition: \"changekind:NO_CHANGE OR changekind:NO_CODE_CHANGE\")\n","accounts_in_message":[],"_revision_number":3},{"id":"39a78eed8b732fdabd9b424789a98cbd1203f285","author":{"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},"date":"2023-07-25 20:48:42.000000000","message":"Patch Set 3: CI-Build+1","accounts_in_message":[],"_revision_number":3},{"id":"2ae224805ba63dc4d4381d2bddd27f1bf7738870","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2023-07-25 20:48:50.000000000","message":"Patch Set 3:\n\nBuild Started https://ci.lttng.org/job/dev_review_lttng-tools_stable-2.13_rootbuild/51/ (1/2)","accounts_in_message":[],"_revision_number":3},{"id":"a637a7106c876f1de35cd5606499ae4c901f3a2f","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2023-07-25 20:48:50.000000000","message":"Patch Set 3:\n\nBuild Started https://ci.lttng.org/job/dev_review_lttng-tools_stable-2.13_build/52/ (2/2)","accounts_in_message":[],"_revision_number":3},{"id":"e26bb9a2d25ba72669c4b2e867b8dd8d18c4ad22","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000002,"name":"jenkins","email":"jenkins@lttng.org","username":"jenkins","avatars":[{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/e3f1da3d4191917309975c0380f40764.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2023-07-25 21:55:41.000000000","message":"Patch Set 3: Verified+2\n\nBuild Successful \n\nhttps://ci.lttng.org/job/dev_review_lttng-tools_stable-2.13_build/52/ : SUCCESS\n\nhttps://ci.lttng.org/job/dev_review_lttng-tools_stable-2.13_rootbuild/51/ : SUCCESS","accounts_in_message":[],"_revision_number":3},{"id":"dc4eba66aee43154f8949055fbab3b1d9ffbd493","tag":"autogenerated:gerrit:merged","author":{"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},"date":"2023-07-27 17:50:33.000000000","message":"Change has been successfully pushed.","accounts_in_message":[],"_revision_number":4}],"current_revision_number":4,"current_revision":"3faa1e3d9bb6b6cd1fa370c206614af9061815be","revisions":{"1d58f067138dba3e52c880feeb6b9826322982e6":{"kind":"REWORK","_number":1,"created":"2023-06-06 21:37:23.000000000","uploader":{"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},"ref":"refs/changes/18/10218/1","fetch":{"anonymous http":{"url":"https://review.lttng.org/lttng-tools","ref":"refs/changes/18/10218/1","commands":{"Branch":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/1 \u0026\u0026 git checkout -b change-10218 FETCH_HEAD","Checkout":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.lttng.org/lttng-tools refs/changes/18/10218/1","Reset To":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/1 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"a989d3e5aea516e321ebd6e021e7c3dedf215a21","subject":"Fix: truncated len in lttng_event_rule_user_tracepoint_serialize()","web_links":[{"name":"gitweb","tooltip":"Open in GitWeb","url":"/gitweb?p\u003dlttng-tools.git;a\u003dcommit;h\u003da989d3e5aea516e321ebd6e021e7c3dedf215a21"}]}],"author":{"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","date":"2023-05-17 17:41:03.000000000","tz":-240},"committer":{"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","date":"2023-06-06 21:36:59.000000000","tz":-240},"subject":"Fix: sessiond: bad fd used while rotating exiting app\u0027s buffers","message":"Fix: sessiond: bad fd used while rotating exiting app\u0027s buffers\n\nIssue observed\n--------------\n\nFrom bug #1372:\n\nWe are observing seemingly random crashes in the LTTng consumer daemon\nwhen tracing a C++ application with LTTng-UST. Our workload has a single\nprintf-like tracepoint, where each string is in the order of 1kb and the\ntotal output is around 30MB/s.\n\nLTTng is set up with a single session and channel enabling this\ntracepoint, and we enabled rotation with a maximum size of 100MB or\nevery 30 seconds. We are periodically starting new traced processes and\nthe system runs close to 100% CPU load. This ran on an AWS\nGraviton2 (ARM) instance with CentOS 7 and a 5.4 kernel, using LTTng-UST\n2.13.5 and LTTng-tools 2.13.8.\n\nThe first reported error is a write to a bad file descriptor (-1),\napparently when waking up the metadata poll thread during a rotation.\n\nCause\n-----\n\nInspecting the logs, we see that the metadata channel with key 574 has a\nnegative poll fd write end which causes the write in\nconsumer_metadata_wakeup_pipe to fail because of an invalid file\ndescriptor:\n\n  DBG1 - 15:12:13.271001175 [6593/6605]: Waking up metadata poll thread (writing to pipe): channel name \u003d \u0027metadata\u0027, channel key \u003d 574 (in consumer_metadata_wakeup_pipe() at consumer.c:888)\n  DBG3 - 15:12:13.271010093 [6593/6605]: write() fd \u003d -1 (in consumer_metadata_wakeup_pipe() at consumer.c:892)\n  PERROR - 15:12:13.271014655 [6593/6605]: Failed to write to UST metadata pipe while attempting to wake-up the metadata poll thread: Bad file descriptor (in consumer_metadata_wakeup_pipe() at consumer.c:907)\n  Error: Failed to dump the metadata cache\n  Error: Rotate channel failed\n\nMeanwhile, a lot of applications seem to be unregistering. Notably, the\napplication associated with that metadata channel it being torn down.\n\nLeading up to the use of a bad file descriptor, the chain of events is:\n\n1) The \"rotation\" thread starts to issue \"Consumer rotate channel\" on\n   key 574 (@ `15:12:12.865621802`), but blocks on the consumer socket\n   lock. We can deduce this from the fact that thread \"6605\" in the\n   consumer wakes up to process an unrelated command originating from the\n   same socket.\n\n   We don\u0027t see that command being issued by the session daemon, most\n   likely because it occurs just before the captured logs start. All\n   call sites that use this socket take the socket lock, issue their\n   command, wait for a reply, and release the socket lock.\n\n2) The application unregisters (@ `15:12:13.269722736`). The\n   `registry_session`, which owns the metadata contents, is destroyed\n   during `delete_ust_app_session` which is done directly as a consequence\n   of the app unregistration (through a deferred RCU call), see\n   `ust_app_unregister`.\n\n   This is problematic since the consumer will request the metadata during\n   the rotation of the metadata channel. In the logs, we can see that\n   the \"close_metadata\" command blocks on the consumer socket lock.\n   However, the problem occurs when the `manage-apps` acquires the lock\n   before the \"rotation\" thread. In this instance, the \"close-metadata\"\n   command is performed by the consumer daemon, closing the metadata\n   poll file descriptor.\n\n3) As the \"close_metadata\" command completes, the rotation thread\n   successfully acquires the socket lock. It is not aware of the\n   unregistration of the application and of the subsequent tear-down of the\n   application, registry, and channels since it was already iterating on\n   the application\u0027s channels.\n\n   The consumer starts to process the channel rotation command (@\n   `15:12:13.270633213`) which fails on the metadata poll fd.\n\nEssentially, we must ensure that the lifetime of metadata\nchannel/streams exceeds any ongoing rotation, and prevent a rotation\nfrom being launched when an application is being torn-down in per-PID\nbuffering mode.\n\nThe problem is fairly hard to reproduce as it requires threads to\nwake-up in the problematic order described above. I don\u0027t have a\nstraight-forward reproducer for the moment.\n\nSolution\n--------\n\nDuring the execution of a rotation on a per-pid session, the session\ndaemon iterates on all applications to rotate their data and metadata\nchannels.\n\nThe `ust_app` itself is correctly protected: it is owned by an RCU HT\n(`ust_app_ht`) and the RCU read lock is acquired as required to protect\nthe lifetime of the storage of `ust_app`. However, there is no way to\nlock an `ust_app` instance itself.\n\nThe rotation command assumes that if it finds the `ust_app`, it will be\nable to rotate all of its channels. This isn\u0027t true: the `ust_app` can\nbe unregistered by the `manage-applications` thread which monitors the\napplication sockets for their deaths in order to teardown the\napplications.\n\nThe `ust_app` doesn\u0027t directly own its channels; they are owned by an\n`ust_app_session` which, itself, has a `lock` mutex. Also, the metadata\nof the application is owned by the \"session registry\", which itself can\nalso be locked.\n\nAt a high-level, we want to ensure that the metadata isn\u0027t closed while\na rotation is being setup. The registry lock could provide this\nguarantee. However, it currently needs to remain unlocked during the\nsetup of the rotation as it is used when providing the metadata to the\nconsumer daemon.\n\nTaking the registry lock over the duration of the setup would result in\na deadlock like so:\n\n- the consumer buffer consumption thread consumed a data buffer and attempts\n  a metadata sync,\n- the command handling thread of the consumer daemon attempts to rotate\n  any stream that is already at its rotation position and locks on the\n  channel lock held by the consumption thread,\n- the metadata sync launches a metadata request against the session\n  daemon which attempts to refresh the metadata contents through the\n  command socket,\n- the command handling thread never services the metadata \"refresh\" sent\n  by the session daemon since it is locked against the same channel as\n  the buffer consumption thread, resulting in a deadlock.\n\nInstead, a different approach is required: extending the lifetime of the\napplication\u0027s channels over the duration of the setup of a rotation.\n\nTo do so, a new object, `lttng_ongoing_rotation`, is introduced which is\nowned by the `ltt_session` being rotated.\n\nThis object acquires a reference to all the channels and that will be\nrotated over the course of the current session rotation.\n\nTaking a reference doesn\u0027t prevent applications from unregistering; it\nsimply defers the reclamation of their buffers to the end of the\nrotation.\n\nAs the rotation completes its setup phase, the references to the buffers are\nreleased, allowing the reclamation of all buffering ressources.\n\nNote that the setup phase of the rotation doesn\u0027t last long so it\nshouldn\u0027t significantly change the observable behaviour in terms of\nmemory usage. The setup phase mostly consists in sampling the\nconsumption/production positions of all buffers in order to establish a\nswitch-over point between the old and new files.\n\nNote\n----\n\nAs far as I can tell, rotations in per-UID buffering mode are not\naffected by this bug since the metadata is never closed by the teardown\nof an application.\n\nThe current diff is an alternative \"minimal\" solution to verify the fix\nby holding a reference to the application only during the launch of its\nchannels\u0027 rotations.\n\nSigned-off-by: Jérémie Galarneau \u003cjeremie.galarneau@efficios.com\u003e\nChange-Id: I8dc1ee45dd00c85556dd70d34a3af4f3a4d4e7cb\n","web_links":[{"name":"gitweb","tooltip":"Open in GitWeb","url":"/gitweb?p\u003dlttng-tools.git;a\u003dcommit;h\u003d1d58f067138dba3e52c880feeb6b9826322982e6"}],"resolve_conflicts_web_links":[{"name":"gitweb","tooltip":"Open in GitWeb","url":"/gitweb?p\u003dlttng-tools.git;a\u003dcommit;h\u003d1d58f067138dba3e52c880feeb6b9826322982e6"}]},"branch":"refs/heads/stable-2.13"},"a65e3a8209a3b9a1fcf4fa265342e370f154500c":{"kind":"NO_CODE_CHANGE","_number":2,"created":"2023-06-27 18:10:43.000000000","uploader":{"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},"ref":"refs/changes/18/10218/2","fetch":{"anonymous http":{"url":"https://review.lttng.org/lttng-tools","ref":"refs/changes/18/10218/2","commands":{"Branch":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/2 \u0026\u0026 git checkout -b change-10218 FETCH_HEAD","Checkout":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.lttng.org/lttng-tools refs/changes/18/10218/2","Reset To":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/2 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"a989d3e5aea516e321ebd6e021e7c3dedf215a21","subject":"Fix: truncated len in lttng_event_rule_user_tracepoint_serialize()","web_links":[{"name":"gitweb","tooltip":"Open in GitWeb","url":"/gitweb?p\u003dlttng-tools.git;a\u003dcommit;h\u003da989d3e5aea516e321ebd6e021e7c3dedf215a21"}]}],"author":{"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","date":"2023-05-17 17:41:03.000000000","tz":-240},"committer":{"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","date":"2023-06-27 18:09:21.000000000","tz":-240},"subject":"Fix: sessiond: bad fd used while rotating exiting app\u0027s buffers","message":"Fix: sessiond: bad fd used while rotating exiting app\u0027s buffers\n\nIssue observed\n--------------\n\nFrom bug #1372:\n\nWe are observing seemingly random crashes in the LTTng consumer daemon\nwhen tracing a C++ application with LTTng-UST. Our workload has a single\nprintf-like tracepoint, where each string is in the order of 1kb and the\ntotal output is around 30MB/s.\n\nLTTng is set up with a single session and channel enabling this\ntracepoint, and we enabled rotation with a maximum size of 100MB or\nevery 30 seconds. We are periodically starting new traced processes and\nthe system runs close to 100% CPU load. This ran on an AWS\nGraviton2 (ARM) instance with CentOS 7 and a 5.4 kernel, using LTTng-UST\n2.13.5 and LTTng-tools 2.13.8.\n\nThe first reported error is a write to a bad file descriptor (-1),\napparently when waking up the metadata poll thread during a rotation.\n\nCause\n-----\n\nInspecting the logs, we see that the metadata channel with key 574 has a\nnegative poll fd write end which causes the write in\nconsumer_metadata_wakeup_pipe to fail because of an invalid file\ndescriptor:\n\n  DBG1 - 15:12:13.271001175 [6593/6605]: Waking up metadata poll thread (writing to pipe): channel name \u003d \u0027metadata\u0027, channel key \u003d 574 (in consumer_metadata_wakeup_pipe() at consumer.c:888)\n  DBG3 - 15:12:13.271010093 [6593/6605]: write() fd \u003d -1 (in consumer_metadata_wakeup_pipe() at consumer.c:892)\n  PERROR - 15:12:13.271014655 [6593/6605]: Failed to write to UST metadata pipe while attempting to wake-up the metadata poll thread: Bad file descriptor (in consumer_metadata_wakeup_pipe() at consumer.c:907)\n  Error: Failed to dump the metadata cache\n  Error: Rotate channel failed\n\nMeanwhile, a lot of applications seem to be unregistering. Notably, the\napplication associated with that metadata channel is being torn down.\n\nLeading up to the use of a bad file descriptor, the chain of events is:\n\n1) The \"rotation\" thread starts to issue \"Consumer rotate channel\" on\n   key 574 (@ `15:12:12.865621802`), but blocks on the consumer socket\n   lock. We can deduce this from the fact that thread \"6605\" in the\n   consumer wakes up to process an unrelated command originating from the\n   same socket.\n\n   We don\u0027t see that command being issued by the session daemon, most\n   likely because it occurs just before the captured logs start. All\n   call sites that use this socket take the socket lock, issue their\n   command, wait for a reply, and release the socket lock.\n\n2) The application unregisters (@ `15:12:13.269722736`). The\n   `registry_session`, which owns the metadata contents, is destroyed\n   during `delete_ust_app_session` which is done directly as a consequence\n   of the app unregistration (through a deferred RCU call), see\n   `ust_app_unregister`.\n\n   This is problematic since the consumer will request the metadata during\n   the rotation of the metadata channel. In the logs, we can see that\n   the \"close_metadata\" command blocks on the consumer socket lock.\n   However, the problem occurs when the `manage-apps` acquires the lock\n   before the \"rotation\" thread. In this instance, the \"close-metadata\"\n   command is performed by the consumer daemon, closing the metadata\n   poll file descriptor.\n\n3) As the \"close_metadata\" command completes, the rotation thread\n   successfully acquires the socket lock. It is not aware of the\n   unregistration of the application and of the subsequent tear-down of the\n   application, registry, and channels since it was already iterating on\n   the application\u0027s channels.\n\n   The consumer starts to process the channel rotation command (@\n   `15:12:13.270633213`) which fails on the metadata poll fd.\n\nEssentially, we must ensure that the lifetime of metadata\nchannel/streams exceeds any ongoing rotation, and prevent a rotation\nfrom being launched when an application is being torn-down in per-PID\nbuffering mode.\n\nThe problem is fairly hard to reproduce as it requires threads to\nwake-up in the problematic order described above. I don\u0027t have a\nstraight-forward reproducer for the moment.\n\nSolution\n--------\n\nDuring the execution of a rotation on a per-pid session, the session\ndaemon iterates on all applications to rotate their data and metadata\nchannels.\n\nThe `ust_app` itself is correctly protected: it is owned by an RCU HT\n(`ust_app_ht`) and the RCU read lock is acquired as required to protect\nthe lifetime of the storage of `ust_app`. However, there is no way to\nlock an `ust_app` instance itself.\n\nThe rotation command assumes that if it finds the `ust_app`, it will be\nable to rotate all of its channels. This isn\u0027t true: the `ust_app` can\nbe unregistered by the `manage-applications` thread which monitors the\napplication sockets for their deaths in order to teardown the\napplications.\n\nThe `ust_app` doesn\u0027t directly own its channels; they are owned by an\n`ust_app_session` which, itself, has a `lock` mutex. Also, the metadata\nof the application is owned by the \"session registry\", which itself can\nalso be locked.\n\nAt a high-level, we want to ensure that the metadata isn\u0027t closed while\na rotation is being setup. The registry lock could provide this\nguarantee. However, it currently needs to remain unlocked during the\nsetup of the rotation as it is used when providing the metadata to the\nconsumer daemon.\n\nTaking the registry lock over the duration of the setup would result in\na deadlock like so:\n\n- the consumer buffer consumption thread consumed a data buffer and attempts\n  a metadata sync,\n- the command handling thread of the consumer daemon attempts to rotate\n  any stream that is already at its rotation position and locks on the\n  channel lock held by the consumption thread,\n- the metadata sync launches a metadata request against the session\n  daemon which attempts to refresh the metadata contents through the\n  command socket,\n- the command handling thread never services the metadata \"refresh\" sent\n  by the session daemon since it is locked against the same channel as\n  the buffer consumption thread, resulting in a deadlock.\n\nInstead, a different approach is required: extending the lifetime of the\napplication\u0027s channels over the duration of the setup of a rotation.\n\nTo do so, the `ust_app` structure (which represents a registered\napplication) is now reference-counted. A reference is acquired over the\nduration of the rotation\u0027s setup phase. This reference transitively\nholds a reference the application\u0027s tracing buffers.\n\nNote that taking a reference doesn\u0027t prevent applications from\nunregistering; it simply defers the reclamation of their buffers to the\nend of the rotation setup.\n\nAs the rotation completes its setup phase, the references to the\napplication (and thus, its tracing buffers) are released, allowing the\nreclamation of all buffering ressources.\n\nNote that the setup phase of the rotation doesn\u0027t last long so it\nshouldn\u0027t significantly change the observable behaviour in terms of\nmemory usage. The setup phase mostly consists in sampling the\nconsumption/production positions of all buffers in order to establish a\nswitch-over point between the old and new files.\n\nSigned-off-by: Jérémie Galarneau \u003cjeremie.galarneau@efficios.com\u003e\nChange-Id: I8dc1ee45dd00c85556dd70d34a3af4f3a4d4e7cb\n","web_links":[{"name":"gitweb","tooltip":"Open in GitWeb","url":"/gitweb?p\u003dlttng-tools.git;a\u003dcommit;h\u003da65e3a8209a3b9a1fcf4fa265342e370f154500c"}],"resolve_conflicts_web_links":[{"name":"gitweb","tooltip":"Open in GitWeb","url":"/gitweb?p\u003dlttng-tools.git;a\u003dcommit;h\u003da65e3a8209a3b9a1fcf4fa265342e370f154500c"}]},"branch":"refs/heads/stable-2.13"},"7dfd28ce94e8e6b180b6178e13a1b532b734579c":{"kind":"REWORK","_number":3,"created":"2023-07-25 20:47:59.000000000","uploader":{"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},"ref":"refs/changes/18/10218/3","fetch":{"anonymous http":{"url":"https://review.lttng.org/lttng-tools","ref":"refs/changes/18/10218/3","commands":{"Branch":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/3 \u0026\u0026 git checkout -b change-10218 FETCH_HEAD","Checkout":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.lttng.org/lttng-tools refs/changes/18/10218/3","Reset To":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/3 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"edbac916fd984b0b50dc3f7cc352a94cb7f24287","subject":"event-rule: set event rule loglevel to domain specific value when unset","web_links":[{"name":"gitweb","tooltip":"Open in GitWeb","url":"/gitweb?p\u003dlttng-tools.git;a\u003dcommit;h\u003dedbac916fd984b0b50dc3f7cc352a94cb7f24287"}]}],"author":{"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","date":"2023-05-17 17:41:03.000000000","tz":-240},"committer":{"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","date":"2023-07-25 20:46:55.000000000","tz":-240},"subject":"Fix: sessiond: bad fd used while rotating exiting app\u0027s buffers","message":"Fix: sessiond: bad fd used while rotating exiting app\u0027s buffers\n\nIssue observed\n--------------\n\nFrom bug #1372:\n\nWe are observing seemingly random crashes in the LTTng consumer daemon\nwhen tracing a C++ application with LTTng-UST. Our workload has a single\nprintf-like tracepoint, where each string is in the order of 1kb and the\ntotal output is around 30MB/s.\n\nLTTng is set up with a single session and channel enabling this\ntracepoint, and we enabled rotation with a maximum size of 100MB or\nevery 30 seconds. We are periodically starting new traced processes and\nthe system runs close to 100% CPU load. This ran on an AWS\nGraviton2 (ARM) instance with CentOS 7 and a 5.4 kernel, using LTTng-UST\n2.13.5 and LTTng-tools 2.13.8.\n\nThe first reported error is a write to a bad file descriptor (-1),\napparently when waking up the metadata poll thread during a rotation.\n\nCause\n-----\n\nInspecting the logs, we see that the metadata channel with key 574 has a\nnegative poll fd write end which causes the write in\nconsumer_metadata_wakeup_pipe to fail because of an invalid file\ndescriptor:\n\n  DBG1 - 15:12:13.271001175 [6593/6605]: Waking up metadata poll thread (writing to pipe): channel name \u003d \u0027metadata\u0027, channel key \u003d 574 (in consumer_metadata_wakeup_pipe() at consumer.c:888)\n  DBG3 - 15:12:13.271010093 [6593/6605]: write() fd \u003d -1 (in consumer_metadata_wakeup_pipe() at consumer.c:892)\n  PERROR - 15:12:13.271014655 [6593/6605]: Failed to write to UST metadata pipe while attempting to wake-up the metadata poll thread: Bad file descriptor (in consumer_metadata_wakeup_pipe() at consumer.c:907)\n  Error: Failed to dump the metadata cache\n  Error: Rotate channel failed\n\nMeanwhile, a lot of applications seem to be unregistering. Notably, the\napplication associated with that metadata channel is being torn down.\n\nLeading up to the use of a bad file descriptor, the chain of events is:\n\n1) The \"rotation\" thread starts to issue \"Consumer rotate channel\" on\n   key 574 (@ `15:12:12.865621802`), but blocks on the consumer socket\n   lock. We can deduce this from the fact that thread \"6605\" in the\n   consumer wakes up to process an unrelated command originating from the\n   same socket.\n\n   We don\u0027t see that command being issued by the session daemon, most\n   likely because it occurs just before the captured logs start. All\n   call sites that use this socket take the socket lock, issue their\n   command, wait for a reply, and release the socket lock.\n\n2) The application unregisters (@ `15:12:13.269722736`). The\n   `registry_session`, which owns the metadata contents, is destroyed\n   during `delete_ust_app_session` which is done directly as a consequence\n   of the app unregistration (through a deferred RCU call), see\n   `ust_app_unregister`.\n\n   This is problematic since the consumer will request the metadata during\n   the rotation of the metadata channel. In the logs, we can see that\n   the \"close_metadata\" command blocks on the consumer socket lock.\n   However, the problem occurs when the `manage-apps` acquires the lock\n   before the \"rotation\" thread. In this instance, the \"close-metadata\"\n   command is performed by the consumer daemon, closing the metadata\n   poll file descriptor.\n\n3) As the \"close_metadata\" command completes, the rotation thread\n   successfully acquires the socket lock. It is not aware of the\n   unregistration of the application and of the subsequent tear-down of the\n   application, registry, and channels since it was already iterating on\n   the application\u0027s channels.\n\n   The consumer starts to process the channel rotation command (@\n   `15:12:13.270633213`) which fails on the metadata poll fd.\n\nEssentially, we must ensure that the lifetime of metadata\nchannel/streams exceeds any ongoing rotation, and prevent a rotation\nfrom being launched when an application is being torn-down in per-PID\nbuffering mode.\n\nThe problem is fairly hard to reproduce as it requires threads to\nwake-up in the problematic order described above. I don\u0027t have a\nstraight-forward reproducer for the moment.\n\nSolution\n--------\n\nDuring the execution of a rotation on a per-pid session, the session\ndaemon iterates on all applications to rotate their data and metadata\nchannels.\n\nThe `ust_app` itself is correctly protected: it is owned by an RCU HT\n(`ust_app_ht`) and the RCU read lock is acquired as required to protect\nthe lifetime of the storage of `ust_app`. However, there is no way to\nlock an `ust_app` instance itself.\n\nThe rotation command assumes that if it finds the `ust_app`, it will be\nable to rotate all of its channels. This isn\u0027t true: the `ust_app` can\nbe unregistered by the `manage-applications` thread which monitors the\napplication sockets for their deaths in order to teardown the\napplications.\n\nThe `ust_app` doesn\u0027t directly own its channels; they are owned by an\n`ust_app_session` which, itself, has a `lock` mutex. Also, the metadata\nof the application is owned by the \"session registry\", which itself can\nalso be locked.\n\nAt a high-level, we want to ensure that the metadata isn\u0027t closed while\na rotation is being setup. The registry lock could provide this\nguarantee. However, it currently needs to remain unlocked during the\nsetup of the rotation as it is used when providing the metadata to the\nconsumer daemon.\n\nTaking the registry lock over the duration of the setup would result in\na deadlock like so:\n\n- the consumer buffer consumption thread consumed a data buffer and attempts\n  a metadata sync,\n- the command handling thread of the consumer daemon attempts to rotate\n  any stream that is already at its rotation position and locks on the\n  channel lock held by the consumption thread,\n- the metadata sync launches a metadata request against the session\n  daemon which attempts to refresh the metadata contents through the\n  command socket,\n- the command handling thread never services the metadata \"refresh\" sent\n  by the session daemon since it is locked against the same channel as\n  the buffer consumption thread, resulting in a deadlock.\n\nInstead, a different approach is required: extending the lifetime of the\napplication\u0027s channels over the duration of the setup of a rotation.\n\nTo do so, the `ust_app` structure (which represents a registered\napplication) is now reference-counted. A reference is acquired over the\nduration of the rotation\u0027s setup phase. This reference transitively\nholds a reference the application\u0027s tracing buffers.\n\nNote that taking a reference doesn\u0027t prevent applications from\nunregistering; it simply defers the reclamation of their buffers to the\nend of the rotation setup.\n\nAs the rotation completes its setup phase, the references to the\napplication (and thus, its tracing buffers) are released, allowing the\nreclamation of all buffering ressources.\n\nNote that the setup phase of the rotation doesn\u0027t last long so it\nshouldn\u0027t significantly change the observable behaviour in terms of\nmemory usage. The setup phase mostly consists in sampling the\nconsumption/production positions of all buffers in order to establish a\nswitch-over point between the old and new files.\n\nSigned-off-by: Jérémie Galarneau \u003cjeremie.galarneau@efficios.com\u003e\nChange-Id: I8dc1ee45dd00c85556dd70d34a3af4f3a4d4e7cb\n","web_links":[{"name":"gitweb","tooltip":"Open in GitWeb","url":"/gitweb?p\u003dlttng-tools.git;a\u003dcommit;h\u003d7dfd28ce94e8e6b180b6178e13a1b532b734579c"}],"resolve_conflicts_web_links":[{"name":"gitweb","tooltip":"Open in GitWeb","url":"/gitweb?p\u003dlttng-tools.git;a\u003dcommit;h\u003d7dfd28ce94e8e6b180b6178e13a1b532b734579c"}]},"branch":"refs/heads/stable-2.13"},"3faa1e3d9bb6b6cd1fa370c206614af9061815be":{"kind":"TRIVIAL_REBASE","_number":4,"created":"2023-07-27 17:50:33.000000000","uploader":{"_account_id":1000006,"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","username":"jgalar","avatars":[{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/8689763327e5bbda7dec9f18846b60ae.jpg?d\u003dretro\u0026r\u003dr\u0026s\u003d120","height":120}]},"ref":"refs/changes/18/10218/4","fetch":{"anonymous http":{"url":"https://review.lttng.org/lttng-tools","ref":"refs/changes/18/10218/4","commands":{"Branch":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/4 \u0026\u0026 git checkout -b change-10218 FETCH_HEAD","Checkout":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.lttng.org/lttng-tools refs/changes/18/10218/4","Reset To":"git fetch https://review.lttng.org/lttng-tools refs/changes/18/10218/4 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"95671f5349e87cdd2ea6cb47243608e9368ab8d5","subject":"Fix: consumerd: slow metadata push slows down application registration","web_links":[{"name":"gitweb","tooltip":"Open in GitWeb","url":"/gitweb?p\u003dlttng-tools.git;a\u003dcommit;h\u003d95671f5349e87cdd2ea6cb47243608e9368ab8d5"}]}],"author":{"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","date":"2023-05-17 17:41:03.000000000","tz":-240},"committer":{"name":"Jérémie Galarneau","email":"jeremie.galarneau@efficios.com","date":"2023-07-27 17:50:10.000000000","tz":-240},"subject":"Fix: sessiond: bad fd used while rotating exiting app\u0027s buffers","message":"Fix: sessiond: bad fd used while rotating exiting app\u0027s buffers\n\nIssue observed\n--------------\n\nFrom bug #1372:\n\nWe are observing seemingly random crashes in the LTTng consumer daemon\nwhen tracing a C++ application with LTTng-UST. Our workload has a single\nprintf-like tracepoint, where each string is in the order of 1kb and the\ntotal output is around 30MB/s.\n\nLTTng is set up with a single session and channel enabling this\ntracepoint, and we enabled rotation with a maximum size of 100MB or\nevery 30 seconds. We are periodically starting new traced processes and\nthe system runs close to 100% CPU load. This ran on an AWS\nGraviton2 (ARM) instance with CentOS 7 and a 5.4 kernel, using LTTng-UST\n2.13.5 and LTTng-tools 2.13.8.\n\nThe first reported error is a write to a bad file descriptor (-1),\napparently when waking up the metadata poll thread during a rotation.\n\nCause\n-----\n\nInspecting the logs, we see that the metadata channel with key 574 has a\nnegative poll fd write end which causes the write in\nconsumer_metadata_wakeup_pipe to fail because of an invalid file\ndescriptor:\n\n  DBG1 - 15:12:13.271001175 [6593/6605]: Waking up metadata poll thread (writing to pipe): channel name \u003d \u0027metadata\u0027, channel key \u003d 574 (in consumer_metadata_wakeup_pipe() at consumer.c:888)\n  DBG3 - 15:12:13.271010093 [6593/6605]: write() fd \u003d -1 (in consumer_metadata_wakeup_pipe() at consumer.c:892)\n  PERROR - 15:12:13.271014655 [6593/6605]: Failed to write to UST metadata pipe while attempting to wake-up the metadata poll thread: Bad file descriptor (in consumer_metadata_wakeup_pipe() at consumer.c:907)\n  Error: Failed to dump the metadata cache\n  Error: Rotate channel failed\n\nMeanwhile, a lot of applications seem to be unregistering. Notably, the\napplication associated with that metadata channel is being torn down.\n\nLeading up to the use of a bad file descriptor, the chain of events is:\n\n1) The \"rotation\" thread starts to issue \"Consumer rotate channel\" on\n   key 574 (@ `15:12:12.865621802`), but blocks on the consumer socket\n   lock. We can deduce this from the fact that thread \"6605\" in the\n   consumer wakes up to process an unrelated command originating from the\n   same socket.\n\n   We don\u0027t see that command being issued by the session daemon, most\n   likely because it occurs just before the captured logs start. All\n   call sites that use this socket take the socket lock, issue their\n   command, wait for a reply, and release the socket lock.\n\n2) The application unregisters (@ `15:12:13.269722736`). The\n   `registry_session`, which owns the metadata contents, is destroyed\n   during `delete_ust_app_session` which is done directly as a consequence\n   of the app unregistration (through a deferred RCU call), see\n   `ust_app_unregister`.\n\n   This is problematic since the consumer will request the metadata during\n   the rotation of the metadata channel. In the logs, we can see that\n   the \"close_metadata\" command blocks on the consumer socket lock.\n   However, the problem occurs when the `manage-apps` acquires the lock\n   before the \"rotation\" thread. In this instance, the \"close-metadata\"\n   command is performed by the consumer daemon, closing the metadata\n   poll file descriptor.\n\n3) As the \"close_metadata\" command completes, the rotation thread\n   successfully acquires the socket lock. It is not aware of the\n   unregistration of the application and of the subsequent tear-down of the\n   application, registry, and channels since it was already iterating on\n   the application\u0027s channels.\n\n   The consumer starts to process the channel rotation command (@\n   `15:12:13.270633213`) which fails on the metadata poll fd.\n\nEssentially, we must ensure that the lifetime of metadata\nchannel/streams exceeds any ongoing rotation, and prevent a rotation\nfrom being launched when an application is being torn-down in per-PID\nbuffering mode.\n\nThe problem is fairly hard to reproduce as it requires threads to\nwake-up in the problematic order described above. I don\u0027t have a\nstraight-forward reproducer for the moment.\n\nSolution\n--------\n\nDuring the execution of a rotation on a per-pid session, the session\ndaemon iterates on all applications to rotate their data and metadata\nchannels.\n\nThe `ust_app` itself is correctly protected: it is owned by an RCU HT\n(`ust_app_ht`) and the RCU read lock is acquired as required to protect\nthe lifetime of the storage of `ust_app`. However, there is no way to\nlock an `ust_app` instance itself.\n\nThe rotation command assumes that if it finds the `ust_app`, it will be\nable to rotate all of its channels. This isn\u0027t true: the `ust_app` can\nbe unregistered by the `manage-applications` thread which monitors the\napplication sockets for their deaths in order to teardown the\napplications.\n\nThe `ust_app` doesn\u0027t directly own its channels; they are owned by an\n`ust_app_session` which, itself, has a `lock` mutex. Also, the metadata\nof the application is owned by the \"session registry\", which itself can\nalso be locked.\n\nAt a high-level, we want to ensure that the metadata isn\u0027t closed while\na rotation is being setup. The registry lock could provide this\nguarantee. However, it currently needs to remain unlocked during the\nsetup of the rotation as it is used when providing the metadata to the\nconsumer daemon.\n\nTaking the registry lock over the duration of the setup would result in\na deadlock like so:\n\n- the consumer buffer consumption thread consumed a data buffer and attempts\n  a metadata sync,\n- the command handling thread of the consumer daemon attempts to rotate\n  any stream that is already at its rotation position and locks on the\n  channel lock held by the consumption thread,\n- the metadata sync launches a metadata request against the session\n  daemon which attempts to refresh the metadata contents through the\n  command socket,\n- the command handling thread never services the metadata \"refresh\" sent\n  by the session daemon since it is locked against the same channel as\n  the buffer consumption thread, resulting in a deadlock.\n\nInstead, a different approach is required: extending the lifetime of the\napplication\u0027s channels over the duration of the setup of a rotation.\n\nTo do so, the `ust_app` structure (which represents a registered\napplication) is now reference-counted. A reference is acquired over the\nduration of the rotation\u0027s setup phase. This reference transitively\nholds a reference the application\u0027s tracing buffers.\n\nNote that taking a reference doesn\u0027t prevent applications from\nunregistering; it simply defers the reclamation of their buffers to the\nend of the rotation setup.\n\nAs the rotation completes its setup phase, the references to the\napplication (and thus, its tracing buffers) are released, allowing the\nreclamation of all buffering ressources.\n\nNote that the setup phase of the rotation doesn\u0027t last long so it\nshouldn\u0027t significantly change the observable behaviour in terms of\nmemory usage. The setup phase mostly consists in sampling the\nconsumption/production positions of all buffers in order to establish a\nswitch-over point between the old and new files.\n\nSigned-off-by: Jérémie Galarneau \u003cjeremie.galarneau@efficios.com\u003e\nChange-Id: I8dc1ee45dd00c85556dd70d34a3af4f3a4d4e7cb\n","web_links":[{"name":"gitweb","tooltip":"Open in GitWeb","url":"/gitweb?p\u003dlttng-tools.git;a\u003dcommit;h\u003d3faa1e3d9bb6b6cd1fa370c206614af9061815be"}],"resolve_conflicts_web_links":[{"name":"gitweb","tooltip":"Open in GitWeb","url":"/gitweb?p\u003dlttng-tools.git;a\u003dcommit;h\u003d3faa1e3d9bb6b6cd1fa370c206614af9061815be"}]},"branch":"refs/heads/stable-2.13"}},"requirements":[],"submit_records":[],"submit_requirements":[]}