diff --git a/.env-gwc b/.env-gwc index f9a2c6b..4418eed 100644 --- a/.env-gwc +++ b/.env-gwc @@ -1,10 +1,12 @@ # ENV vars recommended for running gateway-conformance tests -USE_LIBP2P=false +USE_LIBP2P=true USE_BITSWAP=true -PORT=8090 # helia-http-gateway should be running here -TRUSTLESS_GATEWAYS=http://127.0.0.1:8080 # Kubo should be running here -DELEGATED_ROUTING_V1_HOST=http://127.0.0.1:8080 # Kubo should be running here -DEBUG='helia-http-gateway*,*helia-fetch*,*helia:trustless-gateway-block-broker*' +PORT="8080" # helia-http-gateway should be running here +KUBO_PORT="8081" # Kubo should be running here +TRUSTLESS_GATEWAYS="http://127.0.0.1:8081" # Kubo should be running here +DELEGATED_ROUTING_V1_HOST="http://127.0.0.1:8081" # Kubo should be running here +# DEBUG='helia-http-gateway*,*helia-fetch*,*helia:trustless-gateway-block-broker*' +DEBUG='helia*,helia*:trace' USE_TRUSTLESS_GATEWAYS=true USE_DELEGATED_ROUTING=true @@ -13,4 +15,15 @@ USE_DELEGATED_ROUTING=true # FILE_BLOCKSTORE_PATH=./data/blockstore # Uncomment the below to see request & response headers in the logs -# ECHO_HEADERS=true +GWC_DOCKER_IMAGE=ghcr.io/ipfs/gateway-conformance:v0.5.0 + +# things pass when we skip all these. +GWC_SKIP="^.*(TestDNSLinkGatewayUnixFSDirectoryListing|TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors|TestGatewayJSONCborAndIPNS|TestGatewayIPNSPath|TestRedirectCanonicalIPNS|TestGatewayCache|TestGatewaySubdomains|TestUnixFSDirectoryListingOnSubdomainGateway|TestRedirectsFileWithIfNoneMatchHeader|TestTar|TestRedirects|TestPathGatewayMiscellaneous|TestGatewayUnixFSFileRanges|TestGatewaySymlink|TestUnixFSDirectoryListing|TestGatewayBlock|IPNS|TestTrustless|TestSubdomainGatewayDNSLinkInlining).*$" + +# re-enable TestPathing +GWC_SKIP="^.*(TestDNSLinkGatewayUnixFSDirectoryListing|TestNativeDag|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors|TestGatewayJSONCborAndIPNS|TestGatewayIPNSPath|TestRedirectCanonicalIPNS|TestGatewayCache|TestGatewaySubdomains|TestUnixFSDirectoryListingOnSubdomainGateway|TestRedirectsFileWithIfNoneMatchHeader|TestTar|TestRedirects|TestPathGatewayMiscellaneous|TestGatewayUnixFSFileRanges|TestGatewaySymlink|TestUnixFSDirectoryListing|TestGatewayBlock|IPNS|TestTrustless|TestSubdomainGatewayDNSLinkInlining).*$" +GWC_GATEWAY_URL="http://helia-http-gateway.localhost" +# GWC_SUBDOMAIN_URL="http://helia-http-gateway.localhost" +# GWC_GATEWAY_URL="http://127.0.0.1:8080" +GWC_GATEWAY_URL="http://host.docker.internal:8080" +GWC_SUBDOMAIN_URL="http://host.docker.internal:8080" diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 615dfe9..4c2ba98 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -101,7 +101,8 @@ jobs: # # only-if-cached: helia-ht does not guarantee local cache, we will adjust upstream test (which was Kubo-specific) # for now disabling these test cases - args: -skip '^.*(DirectoryListing|TestGatewayCache|TestSubdomainGatewayDNSLinkInlining|proxy|TestGatewaySubdomainAndIPNS|TestGatewaySubdomains|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestRedirectCanonicalIPNS|TestGatewayIPNSPath|TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$' + args: -skip '^.*(DirectoryListing|TestGatewayCache|proxy|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestNativeDag|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$' + # args: -skip '^.*(DirectoryListing|TestGatewayCache|TestSubdomainGatewayDNSLinkInlining|proxy|TestGatewaySubdomainAndIPNS|TestGatewaySubdomains|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestRedirectCanonicalIPNS|TestGatewayIPNSPath|TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$' # 7. Upload the results - name: Upload MD summary diff --git a/package.json b/package.json index 162124c..a7fa5a0 100644 --- a/package.json +++ b/package.json @@ -63,10 +63,10 @@ "test:e2e-flame": "concurrently -k -s all -n \"gateway,playwright\" -c \"magenta,blue\" \"npm run start:dev-flame\" \"wait-on 'tcp:$PORT' && npm run test:e2e\"", "test:e2e-doctor": "concurrently -k -s all -n \"gateway,playwright\" -c \"magenta,blue\" \"npm run start:dev-doctor\" \"wait-on 'tcp:$PORT' && npm run test:e2e\"", "test:gwc-kubo": "node ./scripts/kubo-init.js", - "test:gwc-helia": "npm run build && node -r dotenv/config --trace-warnings dist/src/index.js dotenv_config_path=./.env-gwc", - "test:gwc-setup": "concurrently -k -s all -n \"kubo,helia\" -c \"magenta,blue\" \"npm run test:gwc-kubo\" \"wait-on 'tcp:8080' && npm run test:gwc-helia\"", - "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace ghcr.io/ipfs/gateway-conformance:v0.4.2 test --gateway-url='http://helia-http-gateway.localhost' --subdomain-url='http://helia-http-gateway.localhost' --verbose --json gwc-report.json --specs subdomain-ipns-gateway,subdomain-ipfs-gateway -- -timeout 30m", - "test:gwc": "concurrently -k -s all -n \"kubo&helia,gateway-conformance\" -c \"magenta,blue\" \"npm run test:gwc-setup\" \"wait-on 'tcp:8090' && npm run test:gwc-execute\"", + "test:gwc-helia": "wait-on \"tcp:$KUBO_PORT\" && npm run start:dev-trace", + "test:gwc-setup": "concurrently -k -s all -n \"kubo,helia\" -c \"magenta,blue\" \"npm run test:gwc-kubo\" \"npm run test:gwc-helia\"", + "test:gwc-execute": "docker run --network host -v $PWD:/workspace -w /workspace $GWC_DOCKER_IMAGE test --gateway-url=$GWC_GATEWAY_URL --subdomain-url=$GWC_SUBDOMAIN_URL --verbose --json gwc-report.json $GWC_SPECS -- -test.skip=\"$GWC_SKIP\" -timeout 30m", + "test:gwc": "concurrently -k -s all -n \"kubo,helia,gateway-conformance\" -c \"magenta,blue,green\" \"npm run test:gwc-kubo\" \"npm run test:gwc-helia\" \"wait-on 'tcp:$PORT' && npm run test:gwc-execute\"", "healthcheck": "node dist/src/healthcheck.js", "debug:until-death": "./debugging/until-death.sh", "debug:test-gateways": "./debugging/test-gateways.sh", diff --git a/scripts/kubo-init.js b/scripts/kubo-init.js index 8912fa8..2e7d461 100644 --- a/scripts/kubo-init.js +++ b/scripts/kubo-init.js @@ -15,6 +15,7 @@ debug.enable('kubo-init*') const kuboFilePath = './scripts/tmp/kubo-path.txt' const GWC_FIXTURES_PATH = `${dirname(kuboFilePath)}/fixtures` +const GWC_DOCKER_IMAGE = process.env.GWC_DOCKER_IMAGE ?? 'ghcr.io/ipfs/gateway-conformance:v0.5.0' async function main () { await $`mkdir -p ${dirname(kuboFilePath)}` @@ -54,7 +55,7 @@ function getExecaOptions ({ cwd, ipfsNsMap, tmpDir }) { async function attemptKuboInit (tmpDir) { const execaOptions = getExecaOptions({ tmpDir }) try { - await $(execaOptions)`npx -y kubo init` + await $(execaOptions)`npx -y kubo init --profile test` log('Kubo initialized at %s', tmpDir) } catch (e) { if (!e.stderr.includes('already exists!')) { @@ -86,8 +87,14 @@ async function writeKuboMetaData () { async function configureKubo (tmpDir) { const execaOptions = getExecaOptions({ tmpDir }) try { - await $(execaOptions)`npx -y kubo config Addresses.Gateway /ip4/127.0.0.1/tcp/8080` + await $(execaOptions)`npx -y kubo config Addresses.Gateway /ip4/127.0.0.1/tcp/${process.env.KUBO_PORT ?? 8081}` + await $(execaOptions)`npx -y kubo config --json Bootstrap ${JSON.stringify([])}` + await $(execaOptions)`npx -y kubo config --json Swarm.DisableNatPortMap true` + await $(execaOptions)`npx -y kubo config --json Discovery.MDNS.Enabled false` + await $(execaOptions)`npx -y kubo config --json Gateway.NoFetch true` await $(execaOptions)`npx -y kubo config --json Gateway.ExposeRoutingAPI true` + await $(execaOptions)`npx -y kubo config --json Gateway.HTTPHeaders.Access-Control-Allow-Origin ${JSON.stringify(['*'])}` + await $(execaOptions)`npx -y kubo config --json Gateway.HTTPHeaders.Access-Control-Allow-Methods ${JSON.stringify(['GET', 'POST', 'PUT', 'OPTIONS'])}` log('Kubo configured') } catch (e) { error('Failed to configure Kubo', e) @@ -97,7 +104,7 @@ async function configureKubo (tmpDir) { async function downloadFixtures () { log('Downloading fixtures') try { - await $`docker run -v ${process.cwd()}:/workspace -w /workspace ghcr.io/ipfs/gateway-conformance:v0.4.2 extract-fixtures --directory ${GWC_FIXTURES_PATH} --merged false` + await $`docker run -v ${process.cwd()}:/workspace -w /workspace ${GWC_DOCKER_IMAGE} extract-fixtures --directory ${GWC_FIXTURES_PATH} --merged false` } catch (e) { error('Error downloading fixtures, assuming current or previous success', e) } diff --git a/src/helia-server.ts b/src/helia-server.ts index 9fd708d..8cf5e67 100644 --- a/src/helia-server.ts +++ b/src/helia-server.ts @@ -58,17 +58,6 @@ export class HeliaServer { this.log('heliaServer Started!') this.routes = [ - { - // without this non-wildcard postfixed path, the '/*' route will match first. - path: '/:ns(ipfs|ipns)/:address', // ipns/dnsLink or ipfs/cid - type: 'GET', - handler: async (request, reply): Promise => this.handleEntry({ request, reply }) - }, - { - path: '/:ns(ipfs|ipns)/:address/*', // ipns/dnsLink/relativePath or ipfs/cid/relativePath - type: 'GET', - handler: async (request, reply): Promise => this.handleEntry({ request, reply }) - }, { path: '/api/v0/version', type: 'POST', @@ -106,11 +95,12 @@ export class HeliaServer { path: '/*', type: 'GET', handler: async (request, reply): Promise => { - try { - await this.fetch({ request, reply }) - } catch { - await reply.code(200).send('try /ipfs/ or /ipns/') - } + // try { + await this.fetch({ request, reply }) + // } catch (err) { + // this.log.error('error fetching:', err) + // await reply.code(500).send('try /ipfs/ or /ipns/') + // } } }, { @@ -130,10 +120,7 @@ export class HeliaServer { ] } - /** - * Redirects to the subdomain gateway. - */ - private async handleEntry ({ request, reply }: RouteHandler): Promise { + #redirectUrl ({ request, reply }: RouteHandler): string | null { const { params } = request this.log('fetch request %s', request.url) const { ns: namespace, '*': relativePath, address } = params as EntryParams @@ -141,14 +128,14 @@ export class HeliaServer { this.log('handling entry: ', { address, namespace, relativePath }) if (!USE_SUBDOMAINS) { this.log('subdomains are disabled, fetching without subdomain') - return this.fetch({ request, reply }) + return null } else { this.log('subdomains are enabled, redirecting to subdomain') } const { peerId, cid } = getIpnsAddressDetails(address) if (peerId != null) { - return this.fetch({ request, reply }) + return null } const cidv1Address = cid?.toString() @@ -169,35 +156,17 @@ export class HeliaServer { // finalUrl += encodeURIComponent(`?${new URLSearchParams(request.query).toString()}`) } let encodedDnsLink = address - if (!isInlinedDnsLink(address)) { + if (address != null && !isInlinedDnsLink(address)) { encodedDnsLink = dnsLinkLabelEncoder(address) } - const finalUrl = `${request.protocol}://${cidv1Address ?? encodedDnsLink}.${namespace}.${request.hostname}/${relativePath ?? ''}` - this.log('redirecting to final URL:', finalUrl) - await reply - .headers({ - Location: finalUrl - }) - .code(301) - .send() + return `${request.protocol}://${cidv1Address ?? encodedDnsLink}.${namespace}.${request.hostname}/${relativePath ?? ''}` } #getFullUrlFromFastifyRequest (request: FastifyRequest): string { - let query = '' - if (request.query != null) { - this.log('request.query:', request.query) - const pairs: string[] = [] - Object.keys(request.query).forEach((key: string) => { - const value = (request.query as Record)[key] - pairs.push(`${key}=${value}`) - }) - if (pairs.length > 0) { - query += '?' + pairs.join('&') - } - } + // FYI: request.url includes the query string - return `${request.protocol}://${request.hostname}${request.url}${query}` + return `${request.protocol}://${request.hostname}${request.url}` } #convertVerifiedFetchResponseToFastifyReply = async (verifiedFetchResponse: Response, reply: FastifyReply): Promise => { @@ -223,6 +192,12 @@ export class HeliaServer { for (const [headerName, headerValue] of verifiedFetchResponse.headers.entries()) { headers[headerName] = headerValue } + + const redirectUrl = this.#redirectUrl({ request: reply.request, reply }) + if (redirectUrl != null) { + headers.Location = redirectUrl + } + // Fastify really does not like streams despite what the documentation and github issues say. const reader = verifiedFetchResponse.body.getReader() reply.raw.writeHead(verifiedFetchResponse.status, headers) @@ -285,7 +260,18 @@ export class HeliaServer { const signal = this.#getRequestAwareSignal(request, url) await this.isReady - const resp = await this.heliaFetch(url, { signal, redirect: 'manual' }) + // pass headers from the original request (IncomingHttpHeaders) to HeadersInit + const headers: Record = {} + for (const [headerName, headerValue] of Object.entries(request.headers)) { + if (headerValue != null) { + if (typeof headerValue === 'string') { + headers[headerName] = headerValue + } else { + headers[headerName] = headerValue.join(',') + } + } + } + const resp = await this.heliaFetch(url, { signal, redirect: 'manual', headers }) await this.#convertVerifiedFetchResponseToFastifyReply(resp, reply) }