这段nestjs代码更新数据有几千条,如何使用node的多线程进行优化嘛?

2023-06-16 307 0

async function combinedMethod() {
  const batchSize = 100000; // 每次查询的批量大小
  const startId = 49904812;
  const endId = 133784381;
  const totalIterations = Math.ceil((endId - startId + 1) / batchSize);
  for (let i = 0; i < totalIterations; i++) {
    const batchStartId = startId + i * batchSize;
    const batchEndId = Math.min(batchStartId + batchSize - 1, endId);
    const users = await this.userservice.find({
      where: {
        id: Between(batchStartId, batchEndId),
      },
    });
    const promises = users.map(async (user) => {
      const courtProvinceId = await getCourtProvinceId(user.court_province);
      const courtCityId = await getCourtCityId(user.court_city);
      if (courtProvinceId) {
        user.court_province_id = courtProvinceId;
        await this.userservice.save(user);
      }
      if (courtCityId) {
        user.court_city_id = courtCityId;
        await this.userservice.save(user);
      }
    });
    await Promise.all(promises);
  }
}
async function getCourtProvinceId(courtProvince) {
  const courtProvinces = await this.authser.find({
    where: {
      name: `${courtProvince}省`,
    },
  });
  if (courtProvinces.length > 0) {
    return courtProvinces[0].code;
  } else {
    return null;
  }
}
async function getCourtCityId(courtCity) {
  try {
    const courtCityData = await this.authser.findOne({ where: { name: courtCity } });
    if (courtCityData) {
      return courtCityData.code;
    } else {
      return null;
    }
  } catch (error) {
    // 处理错误
    return null;
  }
}

查询可以合并为一次查询,后续的操作和保存似乎不影响其他数据,用promiseAll处理一下

你的问题主要是在数据库的查询和保存操作的方式,而不在于 Node.js 的单线程模型,有几个点你可以优化:
1.批量保存,
2.你可以预加载courtProvince 和 courtCity
3.优化数据库查询

async function combinedMethod() {
  const batchSize = 100000; // 每次查询的批量大小
  const startId = 49904812;
  const endId = 133784381;
  const totalIterations = Math.ceil((endId - startId + 1) / batchSize);
  const provinceMap = new Map();
  const cityMap = new Map();
  for (let i = 0; i < totalIterations; i++) {
    const batchStartId = startId + i * batchSize;
    const batchEndId = Math.min(batchStartId + batchSize - 1, endId);
    const users = await this.userservice.find({
      where: {
        id: Between(batchStartId, batchEndId),
      },
    });
    for (const user of users) {
      if (!provinceMap.has(user.court_province)) {
        const courtProvinceId = await getCourtProvinceId(user.court_province);
        provinceMap.set(user.court_province, courtProvinceId);
      }
      user.court_province_id = provinceMap.get(user.court_province);
      if (!cityMap.has(user.court_city)) {
        const courtCityId = await getCourtCityId(user.court_city);
        cityMap.set(user.court_city, courtCityId);
      }
      user.court_city_id = cityMap.get(user.court_city);
    }
    await this.userservice.save(users);
  }
}
async function getCourtProvinceId(courtProvince) {
  const courtProvinces = await this.authser.find({
    where: {
      name: `${courtProvince}省`,
    },
  });
  if (courtProvinces.length > 0) {
    return courtProvinces[0].code;
  } else {
    return null;
  }
}
async function getCourtCityId(courtCity) {
  try {
    const courtCityData = await this.authser.findOne({ where: { name: courtCity } });
    if (courtCityData) {
      return courtCityData.code;
    } else {
      return null;
    }
  } catch (error) {
    // 处理错误
    return null;
  }
}

回答

相关文章

nuxt2部署静态化和ssr的时候访问首页先报404再出现首页为什么?
`clip-path` 如何绘制圆角平行四边形呢?
多线程wait方法报错?
VUE 绑定的方法如何直接使用外部函数?
vue2固定定位该怎么做?
谁有redis实现信号量的代码,希望借鉴一下?