如何在节点js中处理100 000个api请求

问题描述

我在这里面临一些严重的问题。我正在使用节点 js 处理 aws lambda。我的任务是将所有 athena 表数据插入到 mailchimp 如果用户已经存在我正在更新该用户如果没有我将用户插入到 mailchimp，但在 athena db 中我们有 100k 数据。 100k 行，每一行是一个人的数据。 问题是它没有将所有用户数据插入mailchimp，而不是插入一些数据，但不是我不知道为什么的所有数据。

 const md5 = require("md5");
    var Mailchimp = require("mailchimp-api-v3");
    var mailchimp = new Mailchimp("b689f064af1165f83d7309795b64095c-us14");
    require("dotenv").config();
    const _ = require("lodash");
    const athena = require("../src/DB/athena");
    const rds = require("../src/DB/rds");
    
    exports.handler = async (event,context,callback) => {
      let list_id = "578a0790b0";
      try {
        const athena_Data = await athena();
        const rds_Data = await rds();
    
        let AudienceData = await Promise.all(
          athena_Data.Items.map(async (members) => {
            if (members.contact_email !== undefined) {
              let data1 = await rds_Data.rows.map((data) => {
                let merge_tags = data.merge_tag;
                let obj = { name: data.athena_column,value: merge_tags };
                return obj;
              });
              var object = data1.reduce(
                (obj,item) => Object.assign(obj,{ [item.name]: item.value }),{}
              );
              let output = {};
              _.forIn(object,(value1,key1) => {
                _.forIn(members,(value,key) => {
                  if (key1 === key) {
                    return (output[value1] = value);
                  }
                });
              });
              let email_hash = md5(members.contact_email);
              await mailchimp
                .get(`/lists/${list_id}/members/${email_hash}`)
                .then(async (result) => {
                  await mailchimp
                    .put(`/lists/${list_id}/members/${email_hash}`,{
                      email_address: members.contact_email,email_type: "text",status: "subscribed",merge_fields: {
                        ...output,},interests: {},language: "",vip: false,location: {
                        latitude: 0,longitude: 0,marketing_permissions: [],ip_signup: "",timestamp_signup: "",ip_opt: "",timestamp_opt: "",tags: [],})
                    .then((response) => {
                      console.log("updated successfully",response);
                    })
                    .catch((err) => {
                      console.log("something went wrong with update",err);
                    });
                })
                .catch(async (err) => {
                  await mailchimp
                    .post(`/lists/${list_id}/members`,})
                    .then((res) => {
                      console.log("inserted successfully",res);
                    })
                    .catch((err) => {
                      console.log("something went wrong with post",err);
                    });
                });
            } else {
              console.log("email does not exist");
            }
          })
        );
        callback(null,"success");
      } catch (e) {
        callback(e,null);
      }
    };

example:
  For refrences or example
        let object={
          address_1: 'MERGE38',birth_date: 'MERGE37',address_2: 'MERGE39',address_3: 'MERGE40',zip: 'MERGE42',state: 'MERGE41',first_name: 'MERGE30',last_name: 'MERGE31',status: 'MERGE35',city: 'MERGE36',person_id: 'MERGE32',age: 'MERGE33'
        }
        let members={
          person_id: 170207,first_name: 'Vladislav1',last_name: 'Sheykhet',address_1: '5188 Foggy Prairie Manor',city: 'Coats',state: 'VA',zip: '244449454.0',sex: 'M',contact_phone: '[phone]',contact_email: '[e-mail]',smoker_status: 'N'
        }
        let output={
          MERGE38: '5188 Foggy Prairie Manor',MERGE42: '244449454.0',MERGE41: 'VA',MERGE30: 'Vladislav1',MERGE31: 'Sheykhet',MERGE36: 'Coats',MERGE32: 170207
        }

解决方法

暂无找到可以解决该程序问题的有效方法，小编努力寻找整理中！

如果你已经找到好的解决方法，欢迎将解决方案带上本链接一起发送给小编。

小编邮箱:dio#foxmail.com (将#修改为@）