instrumentation.patch 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199
  1. --- yaoyao-mobile@3f500e9/YaoYaoAI/App/YaoYaoAIApp.swift
  2. +++ temporary-benchmark/YaoYaoAI/App/YaoYaoAIApp.swift
  3. @@ -689 +689,3 @@
  4. - if let scenario = UITestScenario.current {
  5. + if ProcessInfo.processInfo.environment["LATENCY_URL"] != nil {
  6. + LatencyFixtureView()
  7. + } else if let scenario = UITestScenario.current {
  8. @@ -707,0 +710,37 @@
  9. +
  10. +@MainActor enum LatencyProbe {
  11. + static func mark(_ name: String, _ detail: String = "") {
  12. + guard let run = ProcessInfo.processInfo.environment["LATENCY_RUN"] else { return }
  13. + let row: [String: Any] = ["time": Date().timeIntervalSince1970, "name": name, "detail": detail]
  14. + guard var data = try? JSONSerialization.data(withJSONObject: row) else { return }; data.append(10)
  15. + let url = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0].appendingPathComponent("latency-\(run).jsonl")
  16. + if !FileManager.default.fileExists(atPath: url.path) { FileManager.default.createFile(atPath: url.path, contents: nil) }
  17. + if let handle = try? FileHandle(forWritingTo: url) { try? handle.seekToEnd(); try? handle.write(contentsOf: data); try? handle.close() }
  18. + }
  19. +}
  20. +
  21. +#if DEBUG
  22. +@MainActor private struct LatencyFixtureView: View {
  23. + @State private var client: HermesRESTClient?
  24. + @State private var error = "连接隔离测试服务"
  25. + var body: some View {
  26. + Group {
  27. + if let client { WorkspaceChatView(client: client, accountID: nil, accountName: "隔离测试", onOpenDrawer: {}) }
  28. + else { Text(error) }
  29. + }.task {
  30. + do {
  31. + let configuration = try ServerConfiguration(urlString: ProcessInfo.processInfo.environment["LATENCY_URL"]!)
  32. + let network = HermesNetworkSession.isolated()
  33. + var request = URLRequest(url: configuration.baseURL.appendingPathComponent("auth/password-login"))
  34. + request.httpMethod = "POST"; request.setValue("application/json", forHTTPHeaderField: "Content-Type")
  35. + request.httpBody = Data(#"{"username":"fixture","password":"fixture-pass","provider":"basic"}"#.utf8)
  36. + let (_, response) = try await network.session.data(for: request)
  37. + guard (response as? HTTPURLResponse)?.statusCode == 200 else { error = "夹具登录失败"; return }
  38. + let rest = HermesRESTClient(session: network.session, cookieStorage: network.cookieStorage)
  39. + await rest.configure(server: configuration, authenticationTransport: .cookie)
  40. + client = rest
  41. + } catch { self.error = error.localizedDescription }
  42. + }
  43. + }
  44. +}
  45. +#endif
  46. --- yaoyao-mobile@3f500e9/YaoYaoAI/Features/WorkspaceChat/WorkspaceChatModel.swift
  47. +++ temporary-benchmark/YaoYaoAI/Features/WorkspaceChat/WorkspaceChatModel.swift
  48. @@ -315,0 +316 @@
  49. + LatencyProbe.mark("message_state", "\(message.role)|\(message.status)|\(message.content.count)")
  50. @@ -1059,0 +1061 @@
  51. + LatencyProbe.mark("send_start")
  52. @@ -1061 +1063 @@
  53. - defer { busy = false }
  54. + defer { busy = false; LatencyProbe.mark("send_ready") }
  55. @@ -1087,0 +1090 @@
  56. + LatencyProbe.mark("post_ack")
  57. --- yaoyao-mobile@3f500e9/YaoYaoAI/Features/WorkspaceChat/WorkspaceTranscriptView.swift
  58. +++ temporary-benchmark/YaoYaoAI/Features/WorkspaceChat/WorkspaceTranscriptView.swift
  59. @@ -108,0 +109 @@
  60. + LatencyProbe.mark("view_text", "\(messages.last?.role ?? "")|\(messages.last?.status ?? "")|\(messages.last?.content.count ?? 0)")
  61. --- yaoyao-mobile@3f500e9/YaoYaoAIUITests/LaunchSmokeTests.swift
  62. +++ temporary-benchmark/YaoYaoAIUITests/LaunchSmokeTests.swift
  63. @@ -9,0 +10,45 @@
  64. +
  65. +extension LaunchSmokeTests {
  66. + func testLatencyCompare() throws {
  67. + continueAfterFailure = false
  68. + let configData = try Data(contentsOf: URL(string: "http://127.0.0.1:18959/config")!)
  69. + let config = try JSONSerialization.jsonObject(with: configData) as! [String: [String: String]]
  70. + let cases = [("yaoyao", 0, "short", 6), ("openmausbot", 0, "short", 6),
  71. + ("openmausbot", 160, "short", 6), ("yaoyao", 160, "short", 6),
  72. + ("yaoyao", 160, "long", 1), ("openmausbot", 160, "long", 1)]
  73. + for (product, delay, kind, count) in cases {
  74. + let group = "series-\(product)-\(delay)-\(kind)"
  75. + _ = try Data(contentsOf: URL(string: "http://127.0.0.1:18959/scenario?delay=\(delay)&run=\(group)-setup")!)
  76. + let isYY = product == "yaoyao"
  77. + let settings = config[isYY ? "yy" : "omb"]!
  78. + let app = isYY ? XCUIApplication() : XCUIApplication(bundleIdentifier: "cn.samien.hermes.bot")
  79. + app.launchEnvironment["LATENCY_URL"] = settings["url"]
  80. + app.launchEnvironment["LATENCY_TOKEN"] = settings["token"]
  81. + app.launchEnvironment["LATENCY_RUN"] = group
  82. + app.launchArguments = ["-AppleLanguages", "(zh-Hans)", "-AppleLocale", "zh_CN"]
  83. + app.launch()
  84. + let chat = isYY ? app.descendants(matching: .any)["workspace.conversation.\(settings["conversationId"]!)"] : app.buttons.matching(NSPredicate(format: "label CONTAINS %@", "LatencyBot")).firstMatch
  85. + XCTAssertTrue(chat.waitForExistence(timeout: 20), app.debugDescription)
  86. + chat.tap()
  87. + let input = app.descendants(matching: .any)[isYY ? "workspace.message-input" : "message-input"].firstMatch
  88. + XCTAssertTrue(input.waitForExistence(timeout: 10), app.debugDescription)
  89. + for number in 1...count {
  90. + let run = "\(group)-\(number)"
  91. + _ = try Data(contentsOf: URL(string: "http://127.0.0.1:18959/scenario?delay=\(delay)&run=\(run)")!)
  92. + input.tap()
  93. + input.typeText("[latency:\(kind)-\(run)]")
  94. + let send = app.buttons[isYY ? "workspace.send" : "latency.send"]
  95. + XCTAssertTrue(send.waitForExistence(timeout: 5), app.debugDescription)
  96. + send.tap()
  97. + let expected = "END-\(kind)-\(run)"
  98. + let reply = app.staticTexts.matching(NSPredicate(format: "label CONTAINS %@", expected)).firstMatch
  99. + XCTAssertTrue(reply.waitForExistence(timeout: 30), app.debugDescription)
  100. + if number == count {
  101. + let attachment = XCTAttachment(screenshot: app.screenshot()); attachment.name = run; attachment.lifetime = .keepAlways; add(attachment)
  102. + }
  103. + Thread.sleep(forTimeInterval: 0.7)
  104. + }
  105. + app.terminate()
  106. + }
  107. + }
  108. +}
  109. --- OpenMausBot@a49d2f7d/ios/App/Session.swift
  110. +++ temporary-benchmark/ios/App/Session.swift
  111. @@ -165,0 +166,8 @@
  112. + if let address = ProcessInfo.processInfo.environment["LATENCY_URL"], let url = URL(string: address) {
  113. + let connection = Connection(id: "latency-fixture", name: "隔离测试", host: url.host!, port: url.port!)
  114. + self.connection = connection; self.connections = [connection]
  115. + self.client = CompanionClient(connection: connection, token: ProcessInfo.processInfo.environment["LATENCY_TOKEN"] ?? "")
  116. + UserDefaults.standard.set(true, forKey: "companion.onboarding.welcomeSeen")
  117. + UserDefaults.standard.set(true, forKey: "companion.onboarding.notificationsSeen")
  118. + return
  119. + }
  120. @@ -747,0 +756 @@
  121. + LatencyProbe.mark("frame_applied", "\(state.streaming.values.map(\.count).max() ?? 0)")
  122. @@ -1018,0 +1028 @@
  123. + LatencyProbe.mark("post_ack")
  124. @@ -2198,0 +2209,11 @@
  125. +
  126. +@MainActor enum LatencyProbe {
  127. + static func mark(_ name: String, _ detail: String = "") {
  128. + guard let run = ProcessInfo.processInfo.environment["LATENCY_RUN"] else { return }
  129. + let row: [String: Any] = ["time": Date().timeIntervalSince1970, "name": name, "detail": detail]
  130. + guard var data = try? JSONSerialization.data(withJSONObject: row) else { return }; data.append(10)
  131. + let url = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0].appendingPathComponent("latency-\(run).jsonl")
  132. + if !FileManager.default.fileExists(atPath: url.path) { FileManager.default.createFile(atPath: url.path, contents: nil) }
  133. + if let handle = try? FileHandle(forWritingTo: url) { try? handle.seekToEnd(); try? handle.write(contentsOf: data); try? handle.close() }
  134. + }
  135. +}
  136. --- OpenMausBot@a49d2f7d/ios/App/ChatView.swift
  137. +++ temporary-benchmark/ios/App/ChatView.swift
  138. @@ -307,0 +308 @@
  139. + LatencyProbe.mark("view_message")
  140. @@ -315,0 +317 @@
  141. + LatencyProbe.mark("view_text", "assistant|streaming|\(length)")
  142. @@ -785,0 +788 @@
  143. + LatencyProbe.mark("send_start")
  144. @@ -796,0 +800 @@
  145. + LatencyProbe.mark("send_ready")
  146. @@ -1277,0 +1282 @@
  147. + .accessibilityIdentifier("latency.send")
  148. --- OpenMausBot@a49d2f7d/server/testing/fake-claude-cli.ts
  149. +++ temporary-benchmark/server/testing/fake-claude-cli.ts
  150. @@ -315,0 +316,23 @@
  151. + if (mode === "latency") {
  152. + const marker = /\[latency:([^\]]+)\]/.exec(promptText(prompt))?.[1] ?? 'short';
  153. + const isLong = marker.includes('long');
  154. + const chunks = isLong ? Array.from({length: 200}, (_, i) => `第${String(i+1).padStart(3,'0')}段。用于比较两个移动端的连续文本显示表现,全部内容来自隔离测试服务。\n\n`)
  155. + : Array.from({length: 40}, (_, i) => `${String(i+1).padStart(2,'0')}测试文字 `);
  156. + let text='';
  157. + setTimeout(() => {
  158. + let index=0;
  159. + const tick=()=>{
  160. + if(index===chunks.length) {
  161. + text+=` END-${marker}`;
  162. + out({type:'stream_event',event:{type:'content_block_delta',delta:{type:'text_delta',text:` END-${marker}`}}});
  163. + out({type:'assistant',message:{content:[{type:'text',text}],usage:{input_tokens:10,output_tokens:100}}});
  164. + out({type:'result',is_error:false,stop_reason:'end_turn',total_cost_usd:0,usage:{input_tokens:10,output_tokens:100}});
  165. + turnRunning=false;finishIfDone();return;
  166. + }
  167. + const part=chunks[index++];text+=part;
  168. + out({type:'stream_event',event:{type:'content_block_delta',delta:{type:'text_delta',text:part}}});
  169. + setTimeout(tick,20);
  170. + }; tick();
  171. + },250);
  172. + return;
  173. + }
  174. --- yaoyao@d4c98d4/tests/fixtures/workspace-server.ts
  175. +++ temporary-benchmark/tests/fixtures/workspace-server.ts
  176. @@ -264,0 +265,23 @@
  177. + if (String(f.params.text).includes('[latency:')) {
  178. + const marker = /\[latency:([^\]]+)\]/.exec(String(f.params.text))?.[1] ?? 'short';
  179. + const isLong = marker.includes('long');
  180. + const chunks = isLong ? Array.from({length: 200}, (_, i) => `第${String(i+1).padStart(3,'0')}段。用于比较两个移动端的连续文本显示表现,全部内容来自隔离测试服务。\n\n`)
  181. + : Array.from({length: 40}, (_, i) => `${String(i+1).padStart(2,'0')}测试文字 `);
  182. + let text = '';
  183. + setTimeout(() => {
  184. + let index = 0;
  185. + const tick = () => {
  186. + if (index === chunks.length) {
  187. + text += ` END-${marker}`;
  188. + event('message.delta', {text:` END-${marker}`}, f.params.session_id);
  189. + stored!.running = false;
  190. + stored!.messages.push({id:randomUUID(), role:'assistant', content:text, timestamp:Date.now()/1000});
  191. + event('message.complete', {text, status:'complete'}, f.params.session_id);
  192. + return;
  193. + }
  194. + const part=chunks[index++]; text+=part; event('message.delta',{text:part},f.params.session_id);
  195. + setTimeout(tick,20);
  196. + }; tick();
  197. + },250);
  198. + return;
  199. + }