computerPool.test.ts 10.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154
  1. // @vitest-environment node
  2. import {afterEach,expect,it,vi} from 'vitest'
  3. import {DatabaseSync} from 'node:sqlite'
  4. import {randomUUID} from 'node:crypto'
  5. import {ComputerPool} from '../../src/runner/computers/pool'
  6. import {ContainerComputerProvider,type ComputerProvider,type ComputerSpecification} from '../../src/runner/computers/container'
  7. const databases:DatabaseSync[]=[]
  8. afterEach(()=>{for(const db of databases.splice(0))db.close()})
  9. const spec=():ComputerSpecification=>({id:randomUUID(),ownerKey:'owner',imageId:`sha256:${'a'.repeat(64)}`})
  10. it('uses the configured idle deadline, resets it after reuse, and never exempts active leases from expiry',async()=>{
  11. const f=fixture();await f.pool.recover();const resource=spec()
  12. f.pool.idleStopMinutes=15
  13. await f.pool.desktop(resource,'start',()=>{})
  14. f.advance(5*60000);await f.pool.expire();expect(f.pool.status('owner')[0]?.status).toBe('idle')
  15. const lease=await f.pool.acquire(resource,'next',()=>{});await f.pool.release(lease,true)
  16. f.advance(15*60000-1);await f.pool.expire();expect(f.pool.status('owner')[0]?.status).toBe('idle')
  17. f.advance(1);await f.pool.expire();expect(f.pool.status('owner')[0]?.status).toBe('free')
  18. f.pool.idleStopMinutes=0;await f.pool.desktop(resource,'start',()=>{})
  19. f.advance(30*24*60*60000);await f.pool.expire();expect(f.pool.status('owner')[0]?.status).toBe('idle')
  20. await f.pool.acquire(resource,'expired-holder',()=>{});f.advance(30001);await f.pool.expire()
  21. expect(f.pool.status('owner')[0]?.status).toBe('free')
  22. await f.pool.desktop(resource,'start',()=>{});await f.pool.close();expect(f.pool.status('owner')[0]?.status).toBe('free')
  23. })
  24. function fixture(limits={concurrent:2,environments:32,ttlMs:30000}){
  25. let clock=100000
  26. const db=new DatabaseSync(':memory:');databases.push(db)
  27. const validate=new ContainerComputerProvider('docker',randomUUID(),'/unused').validateSpecification
  28. const provider={validateSpecification:validate,ensure:vi.fn(async(_spec,authorize)=>{authorize();return {running:true}}),stop:vi.fn(async()=>{}),remove:vi.fn(async()=>{})} as unknown as ComputerProvider
  29. const pool=new ComputerPool(db,provider,limits,()=>clock)
  30. return {db,provider,pool,advance:(value:number)=>{clock+=value}}
  31. }
  32. it('reserves quota before starting, shares an active environment and rejects malformed specs before storage',async()=>{
  33. const f=fixture({concurrent:1,environments:2,ttlMs:30000});await f.pool.recover()
  34. const first=spec();const lease=await f.pool.acquire(first,'task-one',()=>{})
  35. expect(await f.pool.acquire(first,'task-one',()=>{})).toEqual(lease)
  36. // A second holder joins the active shared environment without a new container or quota.
  37. const joined=await f.pool.acquire(first,'task-two',()=>{})
  38. expect(joined.generation).toBe(lease.generation)
  39. expect(f.provider.ensure).toHaveBeenCalledTimes(1)
  40. expect(f.pool.status('owner')[0]?.holderIds).toEqual(expect.arrayContaining(['task-one','task-two']))
  41. await expect(f.pool.acquire(spec(),'task-three',()=>{})).rejects.toMatchObject({code:'computer_quota'})
  42. await expect(f.pool.acquire({...spec(),imageId:'untrusted:latest'},'invalid',()=>{})).rejects.toMatchObject({code:'computer_spec_invalid'})
  43. expect(f.db.prepare('SELECT count(*) AS n FROM computer_environments').get()?.n).toBe(1)
  44. expect(f.pool.status('different-owner')).toEqual([])
  45. // One holder leaving keeps the desktop for the other; the last one stops it.
  46. await f.pool.release(lease)
  47. expect(f.pool.status('owner')[0]?.status).toBe('active')
  48. expect(f.provider.stop).not.toHaveBeenCalled()
  49. await f.pool.release(joined)
  50. expect(f.pool.status('owner')[0]?.status).toBe('free')
  51. expect(f.provider.stop).toHaveBeenCalledTimes(1)
  52. await f.pool.close()
  53. })
  54. it('fences cancellation before an old operation returns and waits for its stop before admitting a successor',async()=>{
  55. const f=fixture();await f.pool.recover();const resource=spec(),lease=await f.pool.acquire(resource,'task-one',()=>{})
  56. let entered=false
  57. const operation=f.pool.use(lease,({signal})=>new Promise<void>((resolve)=>{entered=true;signal.addEventListener('abort',()=>resolve(),{once:true})})).catch(error=>error)
  58. await vi.waitFor(()=>expect(entered).toBe(true))
  59. const released=f.pool.release(lease)
  60. expect(await operation).toMatchObject({code:'computer_lease_stale'})
  61. await released
  62. const next=await f.pool.acquire(resource,'task-two',()=>{})
  63. expect(next.generation).toBeGreaterThan(lease.generation)
  64. expect(()=>f.pool.release(lease)).toThrow('电脑控制权已改变')
  65. expect(f.provider.stop).toHaveBeenCalledTimes(1)
  66. await f.pool.close()
  67. })
  68. it('keeps an unconfirmed stop reserved and recovers it before a new acquisition',async()=>{
  69. const f=fixture();await f.pool.recover();const resource=spec(),lease=await f.pool.acquire(resource,'task',()=>{})
  70. vi.mocked(f.provider.stop).mockRejectedValueOnce(new Error('daemon unavailable'))
  71. await expect(f.pool.release(lease)).rejects.toThrow('daemon unavailable')
  72. expect(f.pool.status('owner')[0]?.status).toBe('uncertain')
  73. await expect(f.pool.acquire(resource,'new-task',()=>{})).rejects.toMatchObject({code:'computer_busy'})
  74. const restarted=new ComputerPool(f.db,f.provider)
  75. await expect(restarted.acquire(resource,'new-task',()=>{})).rejects.toMatchObject({code:'computer_pool_unavailable'})
  76. await restarted.recover()
  77. const next=await restarted.acquire(resource,'new-task',()=>{})
  78. expect(next.generation).toBeGreaterThan(lease.generation)
  79. await restarted.close()
  80. })
  81. it('retries an unconfirmed stop on later sweeps so the environment does not stay blocked',async()=>{
  82. const f=fixture();await f.pool.recover();const resource=spec(),lease=await f.pool.acquire(resource,'task',()=>{})
  83. vi.mocked(f.provider.stop).mockRejectedValueOnce(new Error('daemon unavailable'))
  84. await expect(f.pool.release(lease)).rejects.toThrow('daemon unavailable')
  85. expect(f.pool.status('owner')[0]?.status).toBe('uncertain')
  86. f.advance(29999);await f.pool.expire()
  87. expect(f.pool.status('owner')[0]?.status).toBe('uncertain')
  88. f.advance(1);await f.pool.expire()
  89. expect(f.pool.status('owner')[0]?.status).toBe('free')
  90. const next=await f.pool.acquire(resource,'new-task',()=>{})
  91. expect(next.generation).toBeGreaterThan(lease.generation)
  92. await f.pool.close()
  93. })
  94. it('does not expire a new holder acquired while an earlier environment stop was in flight',async()=>{
  95. const f=fixture();await f.pool.recover();const a=spec(),b=spec()
  96. const first=await f.pool.acquire(a,'first',()=>{}),second=await f.pool.acquire(b,'second',()=>{})
  97. let release!:()=>void,entered=false
  98. const gate=new Promise<void>(resolve=>{release=resolve})
  99. vi.mocked(f.provider.stop).mockImplementation(async resource=>{if(resource.id===a.id){entered=true;await gate}})
  100. f.advance(30001)
  101. const sweep=f.pool.expire()
  102. await vi.waitFor(()=>expect(entered).toBe(true))
  103. await f.pool.release(second)
  104. const replacement=await f.pool.acquire(b,'replacement',()=>{})
  105. release();await sweep
  106. expect(f.pool.status('owner').find(row=>row.environmentId===a.id)?.status).toBe('free')
  107. expect(f.pool.status('owner').find(row=>row.environmentId===b.id)).toMatchObject({status:'active',holderIds:['replacement'],generation:replacement.generation})
  108. expect(()=>f.pool.renew(first)).toThrow()
  109. f.pool.renew(replacement);await f.pool.close()
  110. })
  111. it('rechecks authority and cleans up a creation cancelled while the provider is starting',async()=>{
  112. const f=fixture();await f.pool.recover();const abort=new AbortController()
  113. let release!:()=>void,entered=false
  114. const gate=new Promise<void>(resolve=>{release=resolve})
  115. vi.mocked(f.provider.ensure).mockImplementationOnce(async(_spec,authorize)=>{entered=true;await gate;authorize();return {} as any})
  116. const acquiring=f.pool.acquire(spec(),'starting',()=>{},abort.signal).catch(error=>error)
  117. await vi.waitFor(()=>expect(entered).toBe(true));abort.abort();release()
  118. expect(await acquiring).toBeInstanceOf(Error)
  119. expect(f.pool.status('owner')[0]?.status).toBe('free')
  120. expect(f.provider.stop).toHaveBeenCalledTimes(1)
  121. await f.pool.close()
  122. })
  123. it('rejects changes of account identity and resource configuration for a persistent environment',async()=>{
  124. const f=fixture();await f.pool.recover();const resource=spec(),lease=await f.pool.acquire(resource,'task',()=>{})
  125. await f.pool.release(lease)
  126. for(const change of [{ownerKey:'other'},{memoryMiB:8192}])await expect(f.pool.acquire({...resource,...change},'changed',()=>{})).rejects.toMatchObject({code:'computer_owner_mismatch'})
  127. expect(f.provider.ensure).toHaveBeenCalledTimes(1)
  128. })
  129. it('applies a changed authoritative cwd only after an environment is released',async()=>{
  130. const f=fixture();await f.pool.recover();const resource=spec(),lease=await f.pool.acquire(resource,'task',()=>{})
  131. const updated={...resource,cwd:'/projects/new-location'}
  132. await expect(f.pool.configure(updated,()=>{})).rejects.toMatchObject({code:'computer_busy'})
  133. await f.pool.release(lease)
  134. await f.pool.configure(updated,()=>{})
  135. const next=await f.pool.acquire(updated,'next',()=>{})
  136. expect(next.generation).toBeGreaterThan(lease.generation)
  137. await f.pool.close()
  138. })
  139. it('lets a human takeover join an active environment and fence only its own holder',async()=>{
  140. const f=fixture();await f.pool.recover();const resource=spec(),lease=await f.pool.acquire(resource,'agent',()=>{})
  141. f.provider.inspect=vi.fn(async()=>({id:resource.id,containerId:'same-computer',running:true,workspace:'/fixture',isolation:'container'}))
  142. // The human becomes a second holder of the same live desktop — nothing stops.
  143. const human=await f.pool.acquire(resource,'human:control',()=>{})
  144. expect(human.generation).toBe(lease.generation)
  145. expect(f.pool.status('owner')[0]?.holderIds).toEqual(expect.arrayContaining(['agent','human:control']))
  146. expect(f.provider.stop).not.toHaveBeenCalled()
  147. f.pool.authorize(human);f.pool.authorize(lease)
  148. // Ending the human's control hands the desktop back without stopping it.
  149. await f.pool.release(human,true)
  150. expect(f.pool.status('owner')[0]?.status).toBe('active')
  151. expect(f.pool.status('owner')[0]?.holderIds).toEqual(['agent'])
  152. expect(f.provider.stop).not.toHaveBeenCalled()
  153. await f.pool.close()
  154. })