-
Notifications
You must be signed in to change notification settings - Fork 30
feat(backend): the utilities and services required for the pow feature #5784
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 56 commits
249a09e
d1b36e2
71a07ac
0c86f2a
a464686
228ed56
3b9364e
6db98e4
4962c80
8792a03
c78d546
5e28068
820cdff
5f7e7f9
ed1d241
e24dd02
be5ffed
1409ab4
ae82919
3b2370e
2997b61
9078f2c
a004a9e
c223181
4fd268a
8504980
49ef283
2fedbd0
64c0da1
5ce2ad7
2ea2384
ccdde45
20761a1
4a192a7
f16304a
03306f0
543b545
ceaa96b
7b2a247
3092ca0
957e714
90b0aa8
c33932a
cd2a95b
18b45aa
392f889
4a3632c
ebbc8d2
899db80
c055b79
cb729c4
de27d22
71cd3e8
6efbfcb
66af348
950067f
bbd73cb
4a29858
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -189,3 +189,92 @@ export const inferPostMessageSchema = <T extends z.ZodTypeAny>(dataSchema: T) => | |
msg: z.union([PostMessageRequestSchema, PostMessageResponseSchema]), | ||
data: dataSchema.optional() | ||
}); | ||
|
||
// ----------------------------------------------------------------------------------------------- | ||
// The generic data structures which are required to implement the Short Polling (request-response) | ||
// design pattern used for a simplified communication between the worker(s) and the web application | ||
// The data structure for a request or response to/from a canister call is more or less 1:1 propagated | ||
// by the data structure defined by the schema. | ||
// ----------------------------------------------------------------------------------------------- | ||
/** | ||
* Base schema for all post messages | ||
*/ | ||
export const PostMessageBaseSchema = z.object({ | ||
msg: z.string(), // Message type | ||
requestId: z.string() // Unique identifier for tracking requests and responses | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. we have a pre-existing base There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I do not want to risk the to refactor the other messages since this structure is specifically designed for implementing the short-lived Request-Response Messaging pattern. |
||
}); | ||
|
||
/** | ||
* Base schema for all post message requests | ||
*/ | ||
export const PostMessageRequestBaseSchema = PostMessageBaseSchema.extend({ | ||
type: z.literal('request'), // Specifies this is a request | ||
data: z.unknown().optional() // Optional data payload | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. i'm using a hierarchical inheritance structure for schema. The data is overridden by the actual type. |
||
}); | ||
|
||
export const PostMessageErrorSchema = z.string(); | ||
|
||
/** | ||
* Base schema for all post message responses | ||
*/ | ||
export const PostMessageResponseBaseSchema = PostMessageBaseSchema.extend({ | ||
type: z.literal('response'), | ||
result: z.union([z.object({ Ok: z.unknown() }), z.object({ Err: PostMessageErrorSchema })]) | ||
}); | ||
|
||
export const CreatePowChallengeRequestDataSchema = z.object({}); | ||
|
||
export const PostMessageCreatePowChallengeRequestSchema = PostMessageRequestBaseSchema.extend({ | ||
msg: z.literal('CreatePowChallengeRequest'), | ||
data: CreatePowChallengeRequestDataSchema.optional() | ||
}); | ||
|
||
export const CreatePowChallengeResponseResultSchema = z.object({ | ||
difficulty: z.number(), | ||
start_timestamp_ms: z.bigint(), | ||
expiry_timestamp_ms: z.bigint() | ||
}); | ||
|
||
export const PostMessageCreatePowChallengeResponseSchema = PostMessageResponseBaseSchema.extend({ | ||
msg: z.literal('CreatePowChallengeResponse'), | ||
result: z.union([ | ||
z.object({ Ok: CreatePowChallengeResponseResultSchema }), | ||
z.object({ Err: PostMessageErrorSchema }) | ||
]) | ||
}); | ||
|
||
export const AllowSigningRequestDataSchema = z.object({ | ||
nonce: z.bigint() | ||
}); | ||
|
||
export const PostMessageAllowSigningRequestSchema = PostMessageRequestBaseSchema.extend({ | ||
msg: z.literal('AllowSigningRequest'), | ||
data: AllowSigningRequestDataSchema | ||
}); | ||
|
||
export const ChallengeCompletionSchema = z.object({ | ||
solved_duration_ms: z.bigint(), | ||
next_allowance_ms: z.bigint(), | ||
next_difficulty: z.number(), | ||
current_difficulty: z.number() | ||
}); | ||
|
||
const AllowSigningStatusSchema = z.union([ | ||
z.object({ Skipped: z.null() }), | ||
z.object({ Failed: z.null() }), | ||
z.object({ Executed: z.null() }) | ||
]); | ||
|
||
export const AllowSigningResponseResultSchema = z.object({ | ||
status: AllowSigningStatusSchema, // Use the corrected schema for status | ||
challenge_completion: z.array(z.any()).optional(), // Assuming ChallengeCompletion is modeled correctly elsewhere | ||
allowed_cycles: z.bigint() | ||
}); | ||
|
||
export const PostMessageAllowSigningResponseSchema = PostMessageResponseBaseSchema.extend({ | ||
msg: z.literal('AllowSigningResponse'), | ||
result: z.union([ | ||
z.object({ Ok: AllowSigningResponseResultSchema }), | ||
z.object({ Err: PostMessageErrorSchema }) | ||
]) | ||
}); |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,79 @@ | ||
import type { | ||
AllowSigningRequest, | ||
Result_2 as AllowSigningResult, | ||
Result_6 as CreateChallengeResult | ||
} from '$declarations/backend/backend.did'; | ||
import { allowSigningResult, createPowChallenge } from '$lib/api/backend.api'; | ||
import type { OptionIdentity } from '$lib/types/identity'; | ||
import { hashToHex } from '$lib/utils/crypto.utils'; | ||
|
||
function getTimestampNowMs(): bigint { | ||
return BigInt(Date.now()); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. there is |
||
} | ||
|
||
export const solvePowChallenge = async ({ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. tests for this one? suggestion: put this in another PR |
||
timestamp, | ||
difficulty | ||
}: { | ||
timestamp: bigint; | ||
difficulty: number; | ||
}): Promise<number> => { | ||
if (difficulty <= 0) { | ||
throw new Error('Difficulty must be greater than zero'); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. i18n? |
||
} | ||
|
||
let nonce = 0; | ||
const target = Math.floor(0xffffffff / difficulty); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. maybe some comment here |
||
|
||
// is only used to measure the effective execution time | ||
const startTimestampMs = getTimestampNowMs(); | ||
|
||
while (true) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. let's make it functional |
||
const challengeStr = `${timestamp}.${nonce}`; | ||
const hashHex = await hashToHex(challengeStr); // Using the new hashToHex function | ||
const prefix = parseInt(hashHex.slice(0, 8), 16); // Only consider the first 4 bytes | ||
if (prefix <= target) { | ||
break; | ||
} | ||
nonce++; | ||
} | ||
|
||
const solveTimestampMs = getTimestampNowMs() - startTimestampMs; | ||
console.error(`Pow Challenge solved in ${Number(solveTimestampMs) / 1e3} seconds.`); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why printing an error? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. That was used for debug purpose. I will remove it! |
||
return nonce; | ||
}; | ||
|
||
export const _createPowChallenge = async ({ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. same |
||
identity | ||
}: { | ||
identity: OptionIdentity; | ||
}): Promise<CreateChallengeResult> => { | ||
try { | ||
return await createPowChallenge({ identity }); | ||
} catch (error) { | ||
return { | ||
Err: { | ||
Other: `UnexpectedError: ${error}` | ||
} | ||
}; | ||
} | ||
}; | ||
|
||
export const _allowSigning = async ({ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why this hidden method? |
||
identity, | ||
request | ||
}: { | ||
identity: OptionIdentity; | ||
request?: AllowSigningRequest; | ||
}): Promise<AllowSigningResult> => { | ||
try { | ||
return await allowSigningResult({ identity, request }); | ||
} catch (error) { | ||
// Ensure the `Err` matches the `CreateChallengeError` type | ||
return { | ||
Err: { | ||
Other: `UnexpectedError: ${error}` | ||
} | ||
}; | ||
} | ||
}; |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,10 @@ | ||
import type { | ||
inferPostMessageSchema, | ||
PostMessageAllowSigningRequestSchema, | ||
PostMessageAllowSigningResponseSchema, | ||
PostMessageBaseSchema, | ||
PostMessageCreatePowChallengeRequestSchema, | ||
PostMessageCreatePowChallengeResponseSchema, | ||
PostMessageDataRequestBtcSchema, | ||
PostMessageDataRequestExchangeTimerSchema, | ||
PostMessageDataRequestIcCkBTCUpdateBalanceSchema, | ||
|
@@ -16,10 +22,11 @@ import type { | |
PostMessageDataResponseWalletCleanUpSchema, | ||
PostMessageDataResponseWalletSchema, | ||
PostMessageJsonDataResponseSchema, | ||
PostMessageRequestBaseSchema, | ||
PostMessageResponseBaseSchema, | ||
PostMessageResponseSchema, | ||
PostMessageResponseStatusSchema, | ||
PostMessageSyncStateSchema, | ||
inferPostMessageSchema | ||
PostMessageSyncStateSchema | ||
} from '$lib/schema/post-message.schema'; | ||
import type * as z from 'zod'; | ||
import type { ZodType } from 'zod'; | ||
|
@@ -78,3 +85,20 @@ export type PostMessageDataResponseBTCAddress = z.infer< | |
export type PostMessage<T extends PostMessageDataRequest | PostMessageDataResponse> = z.infer< | ||
ReturnType<typeof inferPostMessageSchema<ZodType<T>>> | ||
>; | ||
|
||
// ----------------------------------------------------------------------------------------------- | ||
// The post message types used for short polling between: | ||
// pow.worker.ts <---> worker.pow.services.ts | ||
// ----------------------------------------------------------------------------------------------- | ||
// Base Types | ||
export type PostMessageBase = z.infer<typeof PostMessageBaseSchema>; | ||
export type PostMessageRequestBase = z.infer<typeof PostMessageRequestBaseSchema>; | ||
export type PostMessageResponseBase = z.infer<typeof PostMessageResponseBaseSchema>; | ||
export type PostMessageCreatePowChallengeRequest = z.infer< | ||
typeof PostMessageCreatePowChallengeRequestSchema | ||
>; | ||
export type PostMessageCreatePowChallengeResponse = z.infer< | ||
typeof PostMessageCreatePowChallengeResponseSchema | ||
>; | ||
export type PostMessageAllowSigningRequest = z.infer<typeof PostMessageAllowSigningRequestSchema>; | ||
export type PostMessageAllowSigningResponse = z.infer<typeof PostMessageAllowSigningResponseSchema>; | ||
Comment on lines
+88
to
+104
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. which one of these types is used in this PR? If they are not needed here, let's remove them (and their schemas if not needed) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
const textEncoder = new TextEncoder(); | ||
|
||
/** | ||
* Hashes the given input string using the SHA-256 algorithm. | ||
*/ | ||
export async function sha256(input: string): Promise<ArrayBuffer> { | ||
return await crypto.subtle.digest('SHA-256', textEncoder.encode(input)); | ||
} | ||
|
||
/** | ||
* Converts an ArrayBuffer to its hexadecimal string representation. | ||
*/ | ||
export function bufferToHex(buffer: ArrayBuffer): string { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. isn't |
||
return Array.from(new Uint8Array(buffer)) | ||
.map((byte) => byte.toString(16).padStart(2, '0')) | ||
.join(''); | ||
} | ||
|
||
/** | ||
* Combines the hashing and hex conversion of a string into a single function. | ||
*/ | ||
export async function hashToHex(input: string): Promise<string> { | ||
const hashBuffer = await sha256(input); | ||
return bufferToHex(hashBuffer); | ||
} | ||
Comment on lines
+6
to
+25
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. don't these functions already exists in some library that we have? P.S. can you use arrow convention for them? for example: |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,68 @@ | ||
import type { PostMessageRequestBase } from '$lib/types/post-message'; | ||
import type { z } from 'zod'; | ||
|
||
const responseHandlers = new Map<string, (data: unknown) => void>(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why data is There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What is this constant? why is it used? why is it not everything handled inside the worker? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is part of the short-lived Request-Response Messaging pattern. It is used to automatically route the request back to the sender. It is private and constant. If I move responseHandlers to pow.worker.ts I would also have to move routeWorkerResponse and sendMessageRequest to pow.worker.ts. |
||
|
||
export function routeWorkerResponse(event: MessageEvent): boolean { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. let's use arrow convention pls There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why the return is a boolean? WDYT of using |
||
//const { type } = event.data; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why this comment? |
||
const { type, requestId } = event.data; | ||
|
||
// Exit immediately if 'type' is missing or not equal to 'response' | ||
if (!type || type !== 'response') { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
console.error("Invalid message type. Expected 'response', but got:", event.data); | ||
return false; | ||
} | ||
|
||
console.warn('Valid data received: ', event.data); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. let's avoid printing warning to console is it necessary? |
||
|
||
const handler = responseHandlers.get(requestId); | ||
if (handler) { | ||
handler(event.data); | ||
responseHandlers.delete(requestId); | ||
return true; | ||
} | ||
console.warn('No handler found for event', requestId); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. let's avoid warnings There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. How do we usually troubleshoot? Is there I dedicated debug logger that can be activated if needed? Should I remove all warnings? |
||
return false; | ||
} | ||
|
||
/** | ||
* Sends a typed request to the worker and awaits the fully typed response envelope (T). | ||
*/ | ||
export function sendMessageRequest<T>({ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. same There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
worker, | ||
msg, | ||
data, | ||
schema | ||
}: { | ||
worker: Worker; | ||
msg: string; | ||
data: object; | ||
schema: z.ZodType<T>; | ||
}): Promise<T> { | ||
const requestId = crypto.randomUUID(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this entire function should be inside the worker. How the other workers handle this? |
||
|
||
// Explicitly type the payload as PostMessageRequestBase | ||
const payload: PostMessageRequestBase = { | ||
msg, | ||
requestId, | ||
type: 'request', | ||
data | ||
}; | ||
|
||
return new Promise((resolve, reject) => { | ||
responseHandlers.set(requestId, (rawResponse: unknown) => { | ||
const parsed = schema.safeParse(rawResponse as PostMessageRequestBase); | ||
if (!parsed.success) { | ||
console.error( | ||
`Invalid response for message '${msg}' (Request ID: ${requestId}):`, | ||
parsed.error.format() | ||
); | ||
reject(parsed.error); | ||
return; | ||
} | ||
resolve(parsed.data); | ||
}); | ||
|
||
worker.postMessage(payload); // Send the typed payload | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. i am not really sure this pertains here... usually the workers that we have handle the messaging in code. For example, have a look at class |
||
}); | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
import { bufferToHex, hashToHex } from '$lib/utils/crypto.utils'; | ||
import { describe, expect, it } from 'vitest'; | ||
|
||
describe('crypto.utils', () => { | ||
describe('bufferToHex', () => { | ||
it('should convert an ArrayBuffer to a hexadecimal string', () => { | ||
const buffer = new Uint8Array([1, 255, 16, 32]).buffer; | ||
const hex = bufferToHex(buffer); | ||
|
||
expect(hex).toBe('01ff1020'); | ||
}); | ||
}); | ||
|
||
describe('hashToHex', () => { | ||
it('should hash a string and return its hex representation', async () => { | ||
const input = 'hash-to-hex-test'; | ||
const hex = await hashToHex(input); | ||
|
||
expect(typeof hex).toBe('string'); | ||
|
||
expect(hex.length).toBe(64); | ||
}); | ||
}); | ||
}); |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
In general, we already have quite a lot
PostMessage
types and schema, we use them for all the workers/schedulers. Is it not possible to use the same? I imagine you are using the same logic, no?