computerPool.test.ts 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156
  1. // @vitest-environment node
  2. import {afterEach,expect,it,vi} from 'vitest'
  3. import {DatabaseSync} from 'node:sqlite'
  4. import {randomUUID} from 'node:crypto'
  5. import {ComputerPool} from '../../src/runner/computers/pool'
  6. import {ContainerComputerProvider,type ComputerProvider,type ComputerSpecification} from '../../src/runner/computers/container'
  7. const databases:DatabaseSync[]=[]
  8. afterEach(()=>{for(const db of databases.splice(0))db.close()})
  9. const spec=():ComputerSpecification=>({id:randomUUID(),ownerKey:'owner',imageId:`sha256:${'a'.repeat(64)}`})
  10. it('uses the configured idle deadline, resets it after reuse, and never exempts active leases from expiry',async()=>{
  11. const f=fixture();await f.pool.recover();const resource=spec()
  12. f.pool.idleStopMinutes=15
  13. await f.pool.desktop(resource,'start',()=>{})
  14. f.advance(5*60000);await f.pool.expire();expect(f.pool.status('owner')[0]?.status).toBe('idle')
  15. const lease=await f.pool.acquire(resource,'next',()=>{})
  16. expect(vi.mocked(f.provider.ensure)).toHaveBeenCalledTimes(2)
  17. await f.pool.release(lease,true)
  18. f.advance(15*60000-1);await f.pool.expire();expect(f.pool.status('owner')[0]?.status).toBe('idle')
  19. f.advance(1);await f.pool.expire();expect(f.pool.status('owner')[0]?.status).toBe('free')
  20. f.pool.idleStopMinutes=0;await f.pool.desktop(resource,'start',()=>{})
  21. f.advance(30*24*60*60000);await f.pool.expire();expect(f.pool.status('owner')[0]?.status).toBe('idle')
  22. await f.pool.acquire(resource,'expired-holder',()=>{});f.advance(30001);await f.pool.expire()
  23. expect(f.pool.status('owner')[0]?.status).toBe('free')
  24. await f.pool.desktop(resource,'start',()=>{});await f.pool.close();expect(f.pool.status('owner')[0]?.status).toBe('free')
  25. })
  26. function fixture(limits={concurrent:2,environments:32,ttlMs:30000}){
  27. let clock=100000
  28. const db=new DatabaseSync(':memory:');databases.push(db)
  29. const validate=new ContainerComputerProvider('docker',randomUUID(),'/unused').validateSpecification
  30. const provider={validateSpecification:validate,ensure:vi.fn(async(_spec,authorize)=>{authorize();return {running:true}}),stop:vi.fn(async()=>{}),remove:vi.fn(async()=>{})} as unknown as ComputerProvider
  31. const pool=new ComputerPool(db,provider,limits,()=>clock)
  32. return {db,provider,pool,advance:(value:number)=>{clock+=value}}
  33. }
  34. it('reserves quota before starting, shares an active environment and rejects malformed specs before storage',async()=>{
  35. const f=fixture({concurrent:1,environments:2,ttlMs:30000});await f.pool.recover()
  36. const first=spec();const lease=await f.pool.acquire(first,'task-one',()=>{})
  37. expect(await f.pool.acquire(first,'task-one',()=>{})).toEqual(lease)
  38. // A second holder joins the active shared environment without a new container or quota.
  39. const joined=await f.pool.acquire(first,'task-two',()=>{})
  40. expect(joined.generation).toBe(lease.generation)
  41. expect(f.provider.ensure).toHaveBeenCalledTimes(1)
  42. expect(f.pool.status('owner')[0]?.holderIds).toEqual(expect.arrayContaining(['task-one','task-two']))
  43. await expect(f.pool.acquire(spec(),'task-three',()=>{})).rejects.toMatchObject({code:'computer_quota'})
  44. await expect(f.pool.acquire({...spec(),imageId:'untrusted:latest'},'invalid',()=>{})).rejects.toMatchObject({code:'computer_spec_invalid'})
  45. expect(f.db.prepare('SELECT count(*) AS n FROM computer_environments').get()?.n).toBe(1)
  46. expect(f.pool.status('different-owner')).toEqual([])
  47. // One holder leaving keeps the desktop for the other; the last one stops it.
  48. await f.pool.release(lease)
  49. expect(f.pool.status('owner')[0]?.status).toBe('active')
  50. expect(f.provider.stop).not.toHaveBeenCalled()
  51. await f.pool.release(joined)
  52. expect(f.pool.status('owner')[0]?.status).toBe('free')
  53. expect(f.provider.stop).toHaveBeenCalledTimes(1)
  54. await f.pool.close()
  55. })
  56. it('fences cancellation before an old operation returns and waits for its stop before admitting a successor',async()=>{
  57. const f=fixture();await f.pool.recover();const resource=spec(),lease=await f.pool.acquire(resource,'task-one',()=>{})
  58. let entered=false
  59. const operation=f.pool.use(lease,({signal})=>new Promise<void>((resolve)=>{entered=true;signal.addEventListener('abort',()=>resolve(),{once:true})})).catch(error=>error)
  60. await vi.waitFor(()=>expect(entered).toBe(true))
  61. const released=f.pool.release(lease)
  62. expect(await operation).toMatchObject({code:'computer_lease_stale'})
  63. await released
  64. const next=await f.pool.acquire(resource,'task-two',()=>{})
  65. expect(next.generation).toBeGreaterThan(lease.generation)
  66. expect(()=>f.pool.release(lease)).toThrow('电脑控制权已改变')
  67. expect(f.provider.stop).toHaveBeenCalledTimes(1)
  68. await f.pool.close()
  69. })
  70. it('keeps an unconfirmed stop reserved and recovers it before a new acquisition',async()=>{
  71. const f=fixture();await f.pool.recover();const resource=spec(),lease=await f.pool.acquire(resource,'task',()=>{})
  72. vi.mocked(f.provider.stop).mockRejectedValueOnce(new Error('daemon unavailable'))
  73. await expect(f.pool.release(lease)).rejects.toThrow('daemon unavailable')
  74. expect(f.pool.status('owner')[0]?.status).toBe('uncertain')
  75. await expect(f.pool.acquire(resource,'new-task',()=>{})).rejects.toMatchObject({code:'computer_busy'})
  76. const restarted=new ComputerPool(f.db,f.provider)
  77. await expect(restarted.acquire(resource,'new-task',()=>{})).rejects.toMatchObject({code:'computer_pool_unavailable'})
  78. await restarted.recover()
  79. const next=await restarted.acquire(resource,'new-task',()=>{})
  80. expect(next.generation).toBeGreaterThan(lease.generation)
  81. await restarted.close()
  82. })
  83. it('retries an unconfirmed stop on later sweeps so the environment does not stay blocked',async()=>{
  84. const f=fixture();await f.pool.recover();const resource=spec(),lease=await f.pool.acquire(resource,'task',()=>{})
  85. vi.mocked(f.provider.stop).mockRejectedValueOnce(new Error('daemon unavailable'))
  86. await expect(f.pool.release(lease)).rejects.toThrow('daemon unavailable')
  87. expect(f.pool.status('owner')[0]?.status).toBe('uncertain')
  88. f.advance(29999);await f.pool.expire()
  89. expect(f.pool.status('owner')[0]?.status).toBe('uncertain')
  90. f.advance(1);await f.pool.expire()
  91. expect(f.pool.status('owner')[0]?.status).toBe('free')
  92. const next=await f.pool.acquire(resource,'new-task',()=>{})
  93. expect(next.generation).toBeGreaterThan(lease.generation)
  94. await f.pool.close()
  95. })
  96. it('does not expire a new holder acquired while an earlier environment stop was in flight',async()=>{
  97. const f=fixture();await f.pool.recover();const a=spec(),b=spec()
  98. const first=await f.pool.acquire(a,'first',()=>{}),second=await f.pool.acquire(b,'second',()=>{})
  99. let release!:()=>void,entered=false
  100. const gate=new Promise<void>(resolve=>{release=resolve})
  101. vi.mocked(f.provider.stop).mockImplementation(async resource=>{if(resource.id===a.id){entered=true;await gate}})
  102. f.advance(30001)
  103. const sweep=f.pool.expire()
  104. await vi.waitFor(()=>expect(entered).toBe(true))
  105. await f.pool.release(second)
  106. const replacement=await f.pool.acquire(b,'replacement',()=>{})
  107. release();await sweep
  108. expect(f.pool.status('owner').find(row=>row.environmentId===a.id)?.status).toBe('free')
  109. expect(f.pool.status('owner').find(row=>row.environmentId===b.id)).toMatchObject({status:'active',holderIds:['replacement'],generation:replacement.generation})
  110. expect(()=>f.pool.renew(first)).toThrow()
  111. f.pool.renew(replacement);await f.pool.close()
  112. })
  113. it('rechecks authority and cleans up a creation cancelled while the provider is starting',async()=>{
  114. const f=fixture();await f.pool.recover();const abort=new AbortController()
  115. let release!:()=>void,entered=false
  116. const gate=new Promise<void>(resolve=>{release=resolve})
  117. vi.mocked(f.provider.ensure).mockImplementationOnce(async(_spec,authorize)=>{entered=true;await gate;authorize();return {} as any})
  118. const acquiring=f.pool.acquire(spec(),'starting',()=>{},abort.signal).catch(error=>error)
  119. await vi.waitFor(()=>expect(entered).toBe(true));abort.abort();release()
  120. expect(await acquiring).toBeInstanceOf(Error)
  121. expect(f.pool.status('owner')[0]?.status).toBe('free')
  122. expect(f.provider.stop).toHaveBeenCalledTimes(1)
  123. await f.pool.close()
  124. })
  125. it('rejects changes of account identity and resource configuration for a persistent environment',async()=>{
  126. const f=fixture();await f.pool.recover();const resource=spec(),lease=await f.pool.acquire(resource,'task',()=>{})
  127. await f.pool.release(lease)
  128. for(const change of [{ownerKey:'other'},{memoryMiB:8192}])await expect(f.pool.acquire({...resource,...change},'changed',()=>{})).rejects.toMatchObject({code:'computer_owner_mismatch'})
  129. expect(f.provider.ensure).toHaveBeenCalledTimes(1)
  130. })
  131. it('applies a changed authoritative cwd only after an environment is released',async()=>{
  132. const f=fixture();await f.pool.recover();const resource=spec(),lease=await f.pool.acquire(resource,'task',()=>{})
  133. const updated={...resource,cwd:'/projects/new-location'}
  134. await expect(f.pool.configure(updated,()=>{})).rejects.toMatchObject({code:'computer_busy'})
  135. await f.pool.release(lease)
  136. await f.pool.configure(updated,()=>{})
  137. const next=await f.pool.acquire(updated,'next',()=>{})
  138. expect(next.generation).toBeGreaterThan(lease.generation)
  139. await f.pool.close()
  140. })
  141. it('lets a human takeover join an active environment and fence only its own holder',async()=>{
  142. const f=fixture();await f.pool.recover();const resource=spec(),lease=await f.pool.acquire(resource,'agent',()=>{})
  143. f.provider.inspect=vi.fn(async()=>({id:resource.id,containerId:'same-computer',running:true,workspace:'/fixture',isolation:'container'}))
  144. // The human becomes a second holder of the same live desktop — nothing stops.
  145. const human=await f.pool.acquire(resource,'human:control',()=>{})
  146. expect(human.generation).toBe(lease.generation)
  147. expect(f.pool.status('owner')[0]?.holderIds).toEqual(expect.arrayContaining(['agent','human:control']))
  148. expect(f.provider.stop).not.toHaveBeenCalled()
  149. f.pool.authorize(human);f.pool.authorize(lease)
  150. // Ending the human's control hands the desktop back without stopping it.
  151. await f.pool.release(human,true)
  152. expect(f.pool.status('owner')[0]?.status).toBe('active')
  153. expect(f.pool.status('owner')[0]?.holderIds).toEqual(['agent'])
  154. expect(f.provider.stop).not.toHaveBeenCalled()
  155. await f.pool.close()
  156. })