diff --git a/README.md b/README.md index 73aa2513e..090440ab4 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-storage/tre | Copy File | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/copyFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/copyFile.js,samples/README.md) | | Copy Old Version Of File. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/copyOldVersionOfFile.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/copyOldVersionOfFile.js,samples/README.md) | | Create a Dual-Region Bucket | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/createBucketWithDualRegion.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/createBucketWithDualRegion.js,samples/README.md) | +| Create a Bucket with object retention enabled. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/createBucketWithObjectRetention.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/createBucketWithObjectRetention.js,samples/README.md) | | Create Bucket With Storage Class and Location. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/createBucketWithStorageClassAndLocation.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/createBucketWithStorageClassAndLocation.js,samples/README.md) | | Create Bucket With Turbo Replication | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/createBucketWithTurboReplication.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/createBucketWithTurboReplication.js,samples/README.md) | | Create New Bucket | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/createNewBucket.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/createNewBucket.js,samples/README.md) | @@ -197,6 +198,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-storage/tre | Set Autoclass | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setAutoclass.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setAutoclass.js,samples/README.md) | | Set Client Endpoint | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setClientEndpoint.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setClientEndpoint.js,samples/README.md) | | Set Event Based Hold | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setEventBasedHold.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setEventBasedHold.js,samples/README.md) | +| Set the object retention policy of a File. | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setObjectRetentionPolicy.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setObjectRetentionPolicy.js,samples/README.md) | | Set Public Access Prevention Enforced | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setPublicAccessPreventionEnforced.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setPublicAccessPreventionEnforced.js,samples/README.md) | | Set Public Access Prevention Inherited | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setPublicAccessPreventionInherited.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setPublicAccessPreventionInherited.js,samples/README.md) | | Set RPO Async Turbo | [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setRPOAsyncTurbo.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setRPOAsyncTurbo.js,samples/README.md) | diff --git a/package.json b/package.json index 01462e990..a49ee9b65 100644 --- a/package.json +++ b/package.json @@ -77,14 +77,12 @@ "@google-cloud/promisify": "^4.0.0", "abort-controller": "^3.0.0", "async-retry": "^1.3.3", - "compressible": "^2.0.12", "duplexify": "^4.1.3", "ent": "^2.2.0", "fast-xml-parser": "^4.3.0", "gaxios": "^6.0.2", "google-auth-library": "^9.6.3", "mime": "^3.0.0", - "mime-types": "^2.0.8", "p-limit": "^3.0.1", "retry-request": "^7.0.0", "teeny-request": "^9.0.0", @@ -97,10 +95,8 @@ "@grpc/grpc-js": "^1.0.3", "@grpc/proto-loader": "^0.7.0", "@types/async-retry": "^1.4.3", - "@types/compressible": "^2.0.0", "@types/ent": "^2.2.1", "@types/mime": "^3.0.0", - "@types/mime-types": "^2.1.0", "@types/mocha": "^9.1.1", "@types/mockery": "^1.4.29", "@types/node": "^20.4.4", diff --git a/samples/README.md b/samples/README.md index 10d611bc0..e6371fe95 100644 --- a/samples/README.md +++ b/samples/README.md @@ -34,6 +34,7 @@ objects to users via direct download. * [Copy File](#copy-file) * [Copy Old Version Of File.](#copy-old-version-of-file.) * [Create a Dual-Region Bucket](#create-a-dual-region-bucket) + * [Create a Bucket with object retention enabled.](#create-a-bucket-with-object-retention-enabled.) * [Create Bucket With Storage Class and Location.](#create-bucket-with-storage-class-and-location.) * [Create Bucket With Turbo Replication](#create-bucket-with-turbo-replication) * [Create New Bucket](#create-new-bucket) @@ -116,6 +117,7 @@ objects to users via direct download. * [Set Autoclass](#set-autoclass) * [Set Client Endpoint](#set-client-endpoint) * [Set Event Based Hold](#set-event-based-hold) + * [Set the object retention policy of a File.](#set-the-object-retention-policy-of-a-file.) * [Set Public Access Prevention Enforced](#set-public-access-prevention-enforced) * [Set Public Access Prevention Inherited](#set-public-access-prevention-inherited) * [Set RPO Async Turbo](#set-rpo-async-turbo) @@ -441,6 +443,25 @@ __Usage:__ +### Create a Bucket with object retention enabled. + +Create a Bucket with object retention enabled. + +View the [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/createBucketWithObjectRetention.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/createBucketWithObjectRetention.js,samples/README.md) + +__Usage:__ + + +`node createBucketWithObjectRetention.js ` + + +----- + + + + ### Create Bucket With Storage Class and Location. Create Bucket With Storage Class and Location. @@ -1879,6 +1900,25 @@ __Usage:__ +### Set the object retention policy of a File. + +Set the object retention policy of a File. + +View the [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setObjectRetentionPolicy.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-storage&page=editor&open_in_editor=samples/setObjectRetentionPolicy.js,samples/README.md) + +__Usage:__ + + +`node setObjectRetentionPolicy.js ` + + +----- + + + + ### Set Public Access Prevention Enforced View the [source code](https://github.com/googleapis/nodejs-storage/blob/main/samples/setPublicAccessPreventionEnforced.js). diff --git a/samples/createBucketWithObjectRetention.js b/samples/createBucketWithObjectRetention.js new file mode 100644 index 000000000..c51ddf7b4 --- /dev/null +++ b/samples/createBucketWithObjectRetention.js @@ -0,0 +1,56 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// sample-metadata: +// title: Create a Bucket with object retention enabled. +// description: Create a Bucket with object retention enabled. +// usage: node createBucketWithObjectRetention.js + +function main(bucketName = 'my-bucket') { + // [START storage_create_bucket_with_object_retention] + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // The ID of your GCS bucket + // const bucketName = 'your-unique-bucket-name'; + + // Imports the Google Cloud client library + const {Storage} = require('@google-cloud/storage'); + + // Creates a client + // The bucket in the sample below will be created in the project associated with this client. + // For more information, please see https://cloud.google.com/docs/authentication/production or https://googleapis.dev/nodejs/storage/latest/Storage.html + const storage = new Storage(); + + async function createBucketWithObjectRetention() { + const [bucket] = await storage.createBucket(bucketName, { + enableObjectRetention: true, + }); + + console.log( + `Created '${bucket.name}' with object retention enabled setting: ${bucket.metadata.objectRetention.mode}` + ); + } + + createBucketWithObjectRetention().catch(console.error); + // [END storage_create_bucket_with_object_retention] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); +main(...process.argv.slice(2)); diff --git a/samples/downloaded.txt b/samples/downloaded.txt deleted file mode 100644 index c57eff55e..000000000 --- a/samples/downloaded.txt +++ /dev/null @@ -1 +0,0 @@ -Hello World! \ No newline at end of file diff --git a/samples/setObjectRetentionPolicy.js b/samples/setObjectRetentionPolicy.js new file mode 100644 index 000000000..fc5c2ab1e --- /dev/null +++ b/samples/setObjectRetentionPolicy.js @@ -0,0 +1,93 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// sample-metadata: +// title: Set the object retention policy of a File. +// description: Set the object retention policy of a File. +// usage: node setObjectRetentionPolicy.js + +function main( + bucketName = 'my-bucket', + destFileName = 'file.txt', + contents = 'this is the file content' +) { + // [START storage_set_object_retention_policy] + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // The ID of your GCS bucket + // const bucketName = 'your-unique-bucket-name'; + + // The new ID for your GCS file + // const destFileName = 'your-new-file-name'; + + // The content to be uploaded in the GCS file + // const contents = 'your file content'; + + // Imports the Google Cloud client library + const {Storage} = require('@google-cloud/storage'); + + // Creates a client + // The bucket in the sample below will be created in the project associated with this client. + // For more information, please see https://cloud.google.com/docs/authentication/production or https://googleapis.dev/nodejs/storage/latest/Storage.html + const storage = new Storage(); + + async function setObjectRetentionPolicy() { + // Get a reference to the bucket + const myBucket = storage.bucket(bucketName); + + // Create a reference to a file object + const file = myBucket.file(destFileName); + + // Save the file data + await file.save(contents); + + // Set the retention policy for the file + const retentionDate = new Date(); + retentionDate.setDate(retentionDate.getDate() + 10); + const [metadata] = await file.setMetadata({ + retention: { + mode: 'Unlocked', + retainUntilTime: retentionDate.toISOString(), + }, + }); + + console.log( + `Retention policy for file ${file.name} was set to: ${metadata.retention.mode}` + ); + + // To modify an existing policy on an unlocked file object, pass in the override parameter + const newRetentionDate = new Date(); + retentionDate.setDate(retentionDate.getDate() + 9); + [metdata] = await file.setMetadata({ + retention: {retainUntilTime: newRetentionDate}, + overrideUnlockedRetention: true, + }); + + console.log( + `Retention policy for file ${file.name} was updated to: ${metadata.retention.retainUntilTime}` + ); + } + + setObjectRetentionPolicy().catch(console.error); + // [END storage_set_object_retention_policy] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); +main(...process.argv.slice(2)); diff --git a/samples/system-test/buckets.test.js b/samples/system-test/buckets.test.js index 3f6b3ac49..744f6b7a8 100644 --- a/samples/system-test/buckets.test.js +++ b/samples/system-test/buckets.test.js @@ -29,6 +29,7 @@ const bucketNameDualRegion = `${samplesTestBucketPrefix}-b`; const bucketNameDualRegionTurbo = `${samplesTestBucketPrefix}-c`; const bucketNameWithClassAndLocation = `${samplesTestBucketPrefix}-d`; const bucketNameAutoclass = `${samplesTestBucketPrefix}-e`; +const bucketNameObjectRetention = `${samplesTestBucketPrefix}-f`; const defaultKmsKeyName = process.env.GOOGLE_CLOUD_KMS_KEY_ASIA; const bucket = storage.bucket(bucketName); const bucketWithClassAndLocation = storage.bucket( @@ -36,6 +37,7 @@ const bucketWithClassAndLocation = storage.bucket( ); const dualRegionBucket = storage.bucket(bucketNameDualRegion); const dualRegionBucketTurbo = storage.bucket(bucketNameDualRegionTurbo); +const objectRetentionBucket = storage.bucket(bucketNameObjectRetention); const PUBLIC_ACCESS_PREVENTION_INHERITED = 'inherited'; const PUBLIC_ACCESS_PREVENTION_ENFORCED = 'enforced'; @@ -417,3 +419,16 @@ it('should delete a bucket', async () => { const [exists] = await bucket.exists(); assert.strictEqual(exists, false); }); + +it('should create a bucket with object retention enabled', async () => { + const output = execSync( + `node createBucketWithObjectRetention.js ${bucketNameObjectRetention}` + ); + console.log(output); + assert.include( + output, + `Created '${bucketNameObjectRetention}' with object retention enabled setting: Enabled` + ); + const [metadata] = await objectRetentionBucket.getMetadata(); + assert.strictEqual(metadata.objectRetention.mode, 'Enabled'); +}); diff --git a/samples/system-test/files.test.js b/samples/system-test/files.test.js index ca55508f3..106c12d1c 100644 --- a/samples/system-test/files.test.js +++ b/samples/system-test/files.test.js @@ -30,6 +30,8 @@ const storage = new Storage(); const cwd = path.join(__dirname, '..'); const bucketName = generateName(); const bucket = storage.bucket(bucketName); +const objectRetentionBucketName = generateName(); +const objectRetentionBucket = storage.bucket(objectRetentionBucketName); const fileContents = 'these-are-my-contents'; const fileName = 'test.txt'; const memoryFileName = 'testmemory.txt'; @@ -576,6 +578,25 @@ describe('file', () => { assert.strictEqual(exists, false); }); }); + + describe('Object Retention', () => { + before(async () => { + await storage.createBucket(objectRetentionBucketName, { + enableObjectRetention: true, + }); + }); + + it('should create a file with unlocked retention and then override it', async () => { + const output = execSync( + `node setObjectRetentionPolicy.js ${objectRetentionBucketName} ${fileName} ${fileContent}` + ); + assert.include(output, 'Retention policy for file'); + const file = objectRetentionBucket.file(fileName); + const [metadata] = await file.getMetadata(); + assert(metadata.retention.retainUntilTime); + assert(metadata.retention.mode.toUpperCase(), 'UNLOCKED'); + }); + }); }); function generateName() { diff --git a/src/bucket.ts b/src/bucket.ts index 599620834..d4321e072 100644 --- a/src/bucket.ts +++ b/src/bucket.ts @@ -29,7 +29,7 @@ import {paginator} from '@google-cloud/paginator'; import {promisifyAll} from '@google-cloud/promisify'; import * as fs from 'fs'; import * as http from 'http'; -import * as mime from 'mime-types'; +import mime from 'mime'; import * as path from 'path'; import pLimit from 'p-limit'; import {promisify} from 'util'; @@ -1625,7 +1625,8 @@ class Bucket extends ServiceObject { callback = callback || util.noop; if (!destinationFile.metadata.contentType) { - const destinationContentType = mime.contentType(destinationFile.name); + const destinationContentType = + mime.getType(destinationFile.name) || undefined; if (destinationContentType) { destinationFile.metadata.contentType = destinationContentType; @@ -1658,6 +1659,7 @@ class Bucket extends ServiceObject { json: { destination: { contentType: destinationFile.metadata.contentType, + contentEncoding: destinationFile.metadata.contentEncoding, }, sourceObjects: (sources as File[]).map(source => { const sourceObject = { diff --git a/src/file.ts b/src/file.ts index 1df546b93..6543316be 100644 --- a/src/file.ts +++ b/src/file.ts @@ -24,7 +24,6 @@ import { } from './nodejs-common/index.js'; import {promisifyAll} from '@google-cloud/promisify'; -import compressible from 'compressible'; import * as crypto from 'crypto'; import * as fs from 'fs'; import mime from 'mime'; @@ -326,6 +325,24 @@ export const STORAGE_POST_POLICY_BASE_URL = 'https://storage.googleapis.com'; */ const GS_URL_REGEXP = /^gs:\/\/([a-z0-9_.-]+)\/(.+)$/; +/** + * @private + */ +const COMPRESSIBLE_MIME_REGEX = new RegExp( + [ + /^text\/|application\/ecmascript|application\/javascript|application\/json/, + /|application\/postscript|application\/rtf|application\/toml|application\/vnd.dart/, + /|application\/vnd.ms-fontobject|application\/wasm|application\/x-httpd-php|application\/x-ns-proxy-autoconfig/, + /|application\/x-sh(?!ockwave-flash)|application\/x-tar|application\/x-virtualbox-hdd|application\/x-virtualbox-ova|application\/x-virtualbox-ovf/, + /|^application\/x-virtualbox-vbox$|application\/x-virtualbox-vdi|application\/x-virtualbox-vhd|application\/x-virtualbox-vmdk/, + /|application\/xml|application\/xml-dtd|font\/otf|font\/ttf|image\/bmp|image\/vnd.adobe.photoshop|image\/vnd.microsoft.icon/, + /|image\/vnd.ms-dds|image\/x-icon|image\/x-ms-bmp|message\/rfc822|model\/gltf-binary|\+json|\+text|\+xml|\+yaml/, + ] + .map(r => r.source) + .join(''), + 'i' +); + export interface FileOptions { crc32cGenerator?: CRC32CValidatorGenerator; encryptionKey?: string | Buffer; @@ -479,6 +496,9 @@ export class RequestError extends Error { } const SEVEN_DAYS = 7 * 24 * 60 * 60; +const GS_UTIL_URL_REGEX = /(gs):\/\/([a-z0-9_.-]+)\/(.+)/g; +const HTTPS_PUBLIC_URL_REGEX = + /(https):\/\/(storage\.googleapis\.com)\/([a-z0-9_.-]+)\/(.+)/g; export enum FileExceptionMessages { EXPIRATION_TIME_NA = 'An expiration time is not available.', @@ -1977,7 +1997,7 @@ class File extends ServiceObject { let gzip = options.gzip; if (gzip === 'auto') { - gzip = compressible(options!.metadata!.contentType || ''); + gzip = COMPRESSIBLE_MIME_REGEX.test(options!.metadata!.contentType || ''); } if (gzip) { @@ -2032,6 +2052,16 @@ class File extends ServiceObject { emitStream.write(chunk, encoding, cb); }, }); + // If the write stream, which is returned to the caller, catches an error we need to make sure that + // at least one of the streams in the pipeline below gets notified so that they + // all get cleaned up / destroyed. + writeStream.once('error', e => { + emitStream.destroy(e); + }); + // If the write stream is closed, cleanup the pipeline below by calling destroy on one of the streams. + writeStream.once('close', () => { + emitStream.destroy(); + }); const transformStreams: Transform[] = []; @@ -2358,6 +2388,35 @@ class File extends ServiceObject { return this; } + /** + * Gets a reference to a Cloud Storage {@link File} file from the provided URL in string format. + * @param {string} publicUrlOrGsUrl the URL as a string. Must be of the format gs://bucket/file + * or https://storage.googleapis.com/bucket/file. + * @param {Storage} storageInstance an instance of a Storage object. + * @param {FileOptions} [options] Configuration options + * @returns {File} + */ + static from( + publicUrlOrGsUrl: string, + storageInstance: Storage, + options?: FileOptions + ): File { + const gsMatches = [...publicUrlOrGsUrl.matchAll(GS_UTIL_URL_REGEX)]; + const httpsMatches = [...publicUrlOrGsUrl.matchAll(HTTPS_PUBLIC_URL_REGEX)]; + + if (gsMatches.length > 0) { + const bucket = new Bucket(storageInstance, gsMatches[0][1]); + return new File(bucket, gsMatches[0][2], options); + } else if (httpsMatches.length > 0) { + const bucket = new Bucket(storageInstance, httpsMatches[0][2]); + return new File(bucket, httpsMatches[0][3], options); + } else { + throw new Error( + 'URL string must be of format gs://bucket/file or https://storage.googleapis.com/bucket/file' + ); + } + } + get(options?: GetFileOptions): Promise>; get(callback: InstanceResponseCallback): void; get(options: GetFileOptions, callback: InstanceResponseCallback): void; diff --git a/src/nodejs-common/service.ts b/src/nodejs-common/service.ts index 0a3111667..8156cd176 100644 --- a/src/nodejs-common/service.ts +++ b/src/nodejs-common/service.ts @@ -271,9 +271,8 @@ export class Service { }; if (reqOpts[GCCL_GCS_CMD_KEY]) { - reqOpts.headers[ - 'x-goog-api-client' - ] += ` gccl-gcs-cmd/${reqOpts[GCCL_GCS_CMD_KEY]}`; + reqOpts.headers['x-goog-api-client'] += + ` gccl-gcs-cmd/${reqOpts[GCCL_GCS_CMD_KEY]}`; } if (reqOpts.shouldReturnStream) { diff --git a/src/resumable-upload.ts b/src/resumable-upload.ts index 049e20c43..5a1f71ccc 100644 --- a/src/resumable-upload.ts +++ b/src/resumable-upload.ts @@ -933,9 +933,8 @@ export class Upload extends Writable { // `Content-Length` for multiple chunk uploads is the size of the chunk, // not the overall object headers['Content-Length'] = bytesToUpload; - headers[ - 'Content-Range' - ] = `bytes ${this.offset}-${endingByte}/${totalObjectSize}`; + headers['Content-Range'] = + `bytes ${this.offset}-${endingByte}/${totalObjectSize}`; } else { headers['Content-Range'] = `bytes ${this.offset}-*/${this.contentLength}`; } diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index 37230389f..8b361b688 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -223,9 +223,8 @@ class XMLMultiPartUploadHelper implements MultiPartUploadHelper { // Prepend command feature to value, if not already there if (!value.includes(GCCL_GCS_CMD_FEATURE.UPLOAD_SHARDED)) { - headers[ - key - ] = `${value} gccl-gcs-cmd/${GCCL_GCS_CMD_FEATURE.UPLOAD_SHARDED}`; + headers[key] = + `${value} gccl-gcs-cmd/${GCCL_GCS_CMD_FEATURE.UPLOAD_SHARDED}`; } } else if (key.toLocaleLowerCase().trim() === 'user-agent') { userAgentFound = true; diff --git a/test/bucket.ts b/test/bucket.ts index 3b81ce170..951cef220 100644 --- a/test/bucket.ts +++ b/test/bucket.ts @@ -22,7 +22,7 @@ import { import assert from 'assert'; import * as fs from 'fs'; import {describe, it, before, beforeEach, after, afterEach} from 'mocha'; -import * as mime from 'mime-types'; +import mime from 'mime'; import pLimit from 'p-limit'; import * as path from 'path'; import proxyquire from 'proxyquire'; @@ -704,7 +704,7 @@ describe('Bucket', () => { destination.request = (reqOpts: DecorateRequestOptions) => { assert.strictEqual( reqOpts.json.destination.contentType, - mime.contentType(destination.name) + mime.getType(destination.name) ); done(); @@ -735,7 +735,7 @@ describe('Bucket', () => { destination.request = (reqOpts: DecorateRequestOptions) => { assert.strictEqual( reqOpts.json.destination.contentType, - mime.contentType(destination.name) + mime.getType(destination.name) ); done(); @@ -745,13 +745,16 @@ describe('Bucket', () => { }); it('should make correct API request', done => { - const sources = [bucket.file('1.txt'), bucket.file('2.txt')]; - const destination = bucket.file('destination.txt'); + const sources = [bucket.file('1.foo'), bucket.file('2.foo')]; + const destination = bucket.file('destination.foo'); destination.request = (reqOpts: DecorateRequestOptions) => { assert.strictEqual(reqOpts.uri, '/compose'); assert.deepStrictEqual(reqOpts.json, { - destination: {contentType: mime.contentType(destination.name)}, + destination: { + contentType: mime.getType(destination.name) || undefined, + contentEncoding: undefined, + }, sourceObjects: [{name: sources[0].name}, {name: sources[1].name}], }); diff --git a/test/file.ts b/test/file.ts index 73de841dd..d0cc0c2ba 100644 --- a/test/file.ts +++ b/test/file.ts @@ -5168,4 +5168,49 @@ describe('File', () => { file.setUserProject(userProject); }); }); + + describe('from', () => { + it('should create a File object from a gs:// formatted URL', () => { + const gsUrl = 'gs://mybucket/myfile'; + const result = File.from(gsUrl, STORAGE); + + assert(result); + assert(result.bucket.name, 'mybucket'); + assert(result.name, 'myfile'); + }); + + it('should create a File object from a gs:// formatted URL including a folder', () => { + const gsUrl = 'gs://mybucket/myfolder/myfile'; + const result = File.from(gsUrl, STORAGE); + + assert(result); + assert(result.bucket.name, 'mybucket'); + assert(result.name, 'myfolder/myfile'); + }); + + it('should create a File object from a https:// formatted URL', () => { + const httpsUrl = 'https://storage.googleapis.com/mybucket/myfile'; + const result = File.from(httpsUrl, STORAGE); + + assert(result); + assert(result.bucket.name, 'mybucket'); + assert(result.name, 'myfile'); + }); + + it('should create a File object from a https:// formatted URL including a folder', () => { + const httpsUrl = + 'https://storage.googleapis.com/mybucket/myfolder/myfile'; + const result = File.from(httpsUrl, STORAGE); + + assert(result); + assert(result.bucket.name, 'mybucket'); + assert(result.name, 'myfolder/myfile'); + }); + + it('should throw an error when invoked with an incorrectly formatted URL', () => { + const invalidUrl = 'https://storage.com/mybucket/myfile'; + + assert.throws(() => File.from(invalidUrl, STORAGE)); + }); + }); });