javascript - 这段nestjs代码更新数据有几千条，如何使用node的多线程进行优化嘛？

Question

async function combinedMethod() {
  const batchSize = 100000; // 每次查询的批量大小
  const startId = 49904812;
  const endId = 133784381;
  const totalIterations = Math.ceil((endId - startId + 1) / batchSize);

  for (let i = 0; i < totalIterations; i++) {
    const batchStartId = startId + i * batchSize;
    const batchEndId = Math.min(batchStartId + batchSize - 1, endId);

    const users = await this.userservice.find({
      where: {
        id: Between(batchStartId, batchEndId),
      },
    });

    const promises = users.map(async (user) => {
      const courtProvinceId = await getCourtProvinceId(user.court_province);
      const courtCityId = await getCourtCityId(user.court_city);

      if (courtProvinceId) {
        user.court_province_id = courtProvinceId;
        await this.userservice.save(user);
      }

      if (courtCityId) {
        user.court_city_id = courtCityId;
        await this.userservice.save(user);
      }
    });

    await Promise.all(promises);
  }
}

async function getCourtProvinceId(courtProvince) {
  const courtProvinces = await this.authser.find({
    where: {
      name: `${courtProvince}省`,
    },
  });

  if (courtProvinces.length > 0) {
    return courtProvinces[0].code;
  } else {
    return null;
  }
}

async function getCourtCityId(courtCity) {
  try {
    const courtCityData = await this.authser.findOne({ where: { name: courtCity } });

    if (courtCityData) {
      return courtCityData.code;
    } else {
      return null;
    }
  } catch (error) {
    // 处理错误
    return null;
  }
}

Answer 1

你的问题主要是在数据库的查询和保存操作的方式，而不在于 Node.js 的单线程模型，有几个点你可以优化：
1.批量保存，
2.你可以预加载courtProvince 和 courtCity
3.优化数据库查询

async function combinedMethod() {
  const batchSize = 100000; // 每次查询的批量大小
  const startId = 49904812;
  const endId = 133784381;
  const totalIterations = Math.ceil((endId - startId + 1) / batchSize);

  const provinceMap = new Map();
  const cityMap = new Map();

  for (let i = 0; i < totalIterations; i++) {
    const batchStartId = startId + i * batchSize;
    const batchEndId = Math.min(batchStartId + batchSize - 1, endId);

    const users = await this.userservice.find({
      where: {
        id: Between(batchStartId, batchEndId),
      },
    });

    for (const user of users) {
      if (!provinceMap.has(user.court_province)) {
        const courtProvinceId = await getCourtProvinceId(user.court_province);
        provinceMap.set(user.court_province, courtProvinceId);
      }
      user.court_province_id = provinceMap.get(user.court_province);

      if (!cityMap.has(user.court_city)) {
        const courtCityId = await getCourtCityId(user.court_city);
        cityMap.set(user.court_city, courtCityId);
      }
      user.court_city_id = cityMap.get(user.court_city);
    }

    await this.userservice.save(users);
  }
}

async function getCourtProvinceId(courtProvince) {
  const courtProvinces = await this.authser.find({
    where: {
      name: `${courtProvince}省`,
    },
  });

  if (courtProvinces.length > 0) {
    return courtProvinces[0].code;
  } else {
    return null;
  }
}

async function getCourtCityId(courtCity) {
  try {
    const courtCityData = await this.authser.findOne({ where: { name: courtCity } });

    if (courtCityData) {
      return courtCityData.code;
    } else {
      return null;
    }
  } catch (error) {
    // 处理错误
    return null;
  }
}

Answer 2

查询可以合并为一次查询，后续的操作和保存似乎不影响其他数据，用promiseAll处理一下