NodeJs AWS 上传脚本

安装配置工具

导入环境变量

export AWS_ACCESS_KEY_ID=*****
export AWS_SECRET_ACCESS_KEY=****

因为S3 SD 不支持在代码中暴露明文信息，需要将秘钥信息放入至环境变量中
！！具体配置环境变量这里不做讲解

> npm install -g @aws-sdk/client-s3
> npm install -g child_process

检查

环境变量是否正常工作

// 这里因为我们将 sdk 安装至 全局目录下
> echo $NODE_PATH
> echo $AWS_ACCESS_KEY_ID
> echo AWS_SECRET_ACCESS_KEY

实现

1、将以下代码保存至: aws.js

// 上传脚本使用方式
// 1、在环境变量中设置
//     export AWS_ACCESS_KEY_ID=****
//  export AWS_SECRET_ACCESS_KEY=0****
// 2、安装全局模块包
// npm install -g @aws-sdk/client-s3
// npm install -g child_process

const {
    S3,
    PutObjectCommand,
    DeleteObjectCommand,
} = require("@aws-sdk/client-s3");
const fs = require("fs");
const path = require("path");
const crypto = require("crypto");
const util = require("util");
const child_process = require("child_process");

// 使用环境变量加载的配置
// https://docs.aws.amazon.com/zh_cn/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html
const s3Client = new S3({ region: "ap-northeast-1" });
const baseUrl = "待设置";
const baseRoute = "static/";

const promisifyReaddir = util.promisify(fs.readdir);
const promisifyStat = util.promisify(fs.stat);
const md5File = "./upload-md5.json";
let uploadQueue = [];
let uploadMd5json = [];

// Acl 设置
// https://stackoverflow.com/questions/65751401/make-s3-object-public-when-uploaded-by-client-s3-typescript-library

const getFileType = (file) => {
    const _file = file.split(".");
    const ext = _file[_file.length - 1];
    switch (ext) {
        case "js":
            return "application/javascript";
        case "css":
            return "text/css";
        case "png":
            return "image/png";
        case "jpg":
            return "image/jpeg";
        case "svg":
            return "image/svg+xml";
        case "ico":
            return "text/html";
        default:
            console.log(ext);
            return "text/plain";
    }
};

// 是否需要上传
const isNeedUpload = async (filePath) => {
    let bufferData = Buffer.from(fs.readFileSync(filePath), "base64");
    const hash = crypto.createHash("md5");
    hash.update(bufferData, "utf8");
    const md5 = hash.digest("hex");
    const isExt = uploadMd5json.includes(md5);
    if (!isExt) {
        uploadMd5json.push(md5);
    }

    return !isExt;
};

const loadMd5Json = async () => {
    const exists = await new Promise((resolve, reject) => {
        fs.exists(md5File, (exists) => {
            resolve(exists);
        });
    });
    if (!exists) {
        await new Promise((resolve, reject) => {
            fs.writeFile(md5File, "[]", (error) => {
                if (error) {
                    throw new Error("创建md5文件失败: ", error);
                }
            });
        });
    }
    uploadMd5json = fs.readFileSync(md5File, "utf8");
    uploadMd5json = JSON.parse(uploadMd5json);
};

const uploadAws = async (file, filekey) => {
    // 检查是否需要上传
    const isNeed = await isNeedUpload(file);
    if (!isNeed) {
        return "no";
    }

    const rootFileKey = baseRoute + filekey;
    const filetype = getFileType(file);
    const bucketParams = {
        Bucket: "tpstatic",
        Key: rootFileKey,
        Body: fs.createReadStream(file),
        ACL: "public-read",
        ContentType: filetype,
    };

    // console.log(filetype);

    // s3Client
    //     .send(new DeleteObjectCommand(bucketParams))
    //     .then(() => {
    //         console.log("删除成功: " + rootFileKey);
    //     })
    //     .catch((error) => {
    //         reject(error);
    //     });

    return new Promise((resolve, reject) => {
        s3Client
            .send(new PutObjectCommand(bucketParams))
            .then(() => {
                resolve(`${baseUrl}${rootFileKey}`);
            })
            .catch((error) => {
                reject(error);
            });
    });
};

async function uploadRunQueue(uploadPath) {
    const dir = await promisifyReaddir(`${uploadPath}`);
    for (let i = 0; i < dir.length; i++) {
        const stat = await promisifyStat(path.resolve(`${uploadPath}`, dir[i]));

        if (stat.isFile()) {
            const filekey = `${uploadPath}/${dir[i]}`;
            const file = path.resolve(`${uploadPath}`, dir[i]);

            uploadQueue.push(
                new Promise((resolve, reject) => {
                    uploadAws(file, filekey).then((url) => {
                        if (url === "no") {
                            console.log("无需上传: " + file);
                        } else {
                            console.log("上传成功:" + url);
                        }
                        resolve(true);
                    });
                })
            );
        } else if (stat.isDirectory()) {
            await uploadRunQueue(`${uploadPath}/${dir[i]}`);
        }
    }
}

const run = async (uploadDir) => {
    await loadMd5Json();
    uploadRunQueue(uploadDir);
    await new Promise((resolve) => {
        console.log("上传进行中 =>>>>");
        setTimeout(async () => {
            await Promise.all(uploadQueue).then((values) => {
                console.log("上传完成 ✅");
                console.log(`已处理个数: ${values.length}`);
            });
            resolve(true);
        }, 500);
    });
    fs.writeFileSync(md5File, JSON.stringify(uploadMd5json));
};

const args = process.argv.slice(2);
if (args.length === 0) {
    console.warn(
        "上传失败: 请设置要上传的目录或者文件, 使用方式可以在当前目录下创建一个文件夹test, 并放置一个文件，再输入 node awsUpload.js test"
    );
    process.exit();
}
const uploadDir = args[0];

run(uploadDir);

2、设置 baseUrl 、baseRoute

baseRoute: 上传的路径
baseUrl: Aws 配置的域名

2、创建文件夹 test

3、执行 aws.js

生成 upload-md5.json
用来防止重复上传

4、执行 aws.js

执行操作

node aws.js test

本文由邓尘锋创作，采用知识共享署名4.0 国际许可协议进行许可
本站文章除注明转载/出处外，均为本站原创或翻译，转载前请务必署名
最后编辑时间为: Feb 17, 2023 at 04:20 pm

NodeJs AWS 上传脚本

相关文档

安装配置工具

导入环境变量

检查

实现

执行操作