NK University

ユーザ用ツール

サイト用ツール

トレース: • scraping
nodejs:scraping

文書の過去の版を表示しています。

スクレイピング

一つ上へ

必要なパッケージをインストールする 

npm install cheerio axios

例

// index.js
 
const axios = require('axios');
const cheerio = require('cheerio');
 
// スクレイピング対象のURL
const url = 'https://announce.wowma.jp/';
 
// Slack Incoming WebhooksのURLを設定
const slackWebhookUrl = '';
 
// Axiosを使用してHTMLを取得
axios.get(url)
  .then(response => {
    // 取得したHTMLをCheerioでパース
    const $ = cheerio.load(response.data);
 
    // スクレイピング対象の要素を指定してデータを取得
    // .modWhiteBox01
    const targetDiv = $('.modWhiteBox01');
 
    // div要素内の uniMainList セレクタ指定して取得
    const innerTargetDiv = targetDiv.find('.uniMainList');
 
    // その中の li 要素を取得
    const liElements = innerTargetDiv.find('li');
 
    const items = [];
 
    liElements.each((index, element) => {
      // 改行で分割し、空白でない行のみを取り出す
      var lines = $(element).text().replace(/\t/g, '').split('\n').filter(line => line.trim() !== '');
 
      const date = lines[0].trim();
      const tag = lines[1].trim();
      const title = lines[2].trim();
      const today = new Date();
      const yesterday = new Date();
      yesterday.setDate(today.getDate() - 1);
      //console.log(yesterday.toISOString().split('T')[0].replace(/-/g, '/'));
      const formattedToday = today.toISOString().split('T')[0].replace(/-/g, '/');
      if (tag.includes('メンテナンス') || tag.includes('障害')) {
        items.push({
        date: date,
        tag: tag,
        title: title
        });
      }
    });
  console.log(items);
 
 
  })
  .catch(error => {
    console.error('エラー:', error);
  });

以下を実行する 

npm run start
// index.js
 
const axios = require('axios');
const cheerio = require('cheerio');
const fs = require('fs');
const csv = require('csv-parser');
const createCsvWriter = require('csv-writer').createObjectCsvWriter;
 
 
// スクレイピング対象のURL
//const url = 'https://www.amazon.co.jp/dp/B0B3LQH6CR'; 
 
const urls = [
'https://www.amazon.co.jp/dp/B0B3LQH6CR',
'https://www.amazon.co.jp/dp/B08BPRKJ6Q?th=1',
'https://www.amazon.co.jp/dp/B003Y8YUVK?th=1',
'https://www.amazon.co.jp/dp/B01N631MK8',
'https://shopdisney.disney.co.jp/goods/4550586678422.html',
'https://shopdisney.disney.co.jp/goods/4550586678415.html',
'https://shopdisney.disney.co.jp/goods/4550586678446.html',
'https://shopdisney.disney.co.jp/goods/4550586678439.html',
'https://shopdisney.disney.co.jp/goods/4550586686120.html',
'https://shopdisney.disney.co.jp/goods/4550586686137.html',
'https://shopdisney.disney.co.jp/goods/4550586686168.html',
'https://shopdisney.disney.co.jp/goods/4550586686151.html',
'https://shopdisney.disney.co.jp/goods/4550586686144.html',
'https://shopdisney.disney.co.jp/goods/4550586678378.html',
'https://shopdisney.disney.co.jp/goods/4550586678392.html',
'https://shopdisney.disney.co.jp/goods/4550424703866.html',
'https://shopdisney.disney.co.jp/goods/4550424703873.html',
'https://shopdisney.disney.co.jp/goods/4550424703859.html?isProductSearch=0&plpPosition=7&guestFacing=%25E3%2581%258A%25E3%2581%2599%25E3%2581%2599%25E3%2582%2581%25E3%2582%25B0%25E3%2583%2583%25E3%2582%25BA%25E7%2589%25B9%25E9%259B%2586-%25E3%2581%25AC%25E3%2581%2584%25E3%2581%2590%25E3%2582%258B%25E3%2581%25BF%25E3%2582%25B7%25E3%2583%25AA%25E3%2583%25BC%25E3%2582%25BA%25E3%2580%258C%25E3%2581%2586%25E3%2582%258B%25E3%2581%25BD%25E3%2581%25A1%25E3%2582%2583%25E3%2581%25A1%25E3%2582%2583%25E3%2582%2593%25E3%2580%258D',
'https://shopdisney.disney.co.jp/goods/4550424862815.html?isProductSearch=0&plpPosition=5&guestFacing=%25E3%2581%258A%25E3%2581%2599%25E3%2581%2599%25E3%2582%2581%25E3%2582%25B0%25E3%2583%2583%25E3%2582%25BA%25E7%2589%25B9%25E9%259B%2586-%25E3%2581%25AC%25E3%2581%2584%25E3%2581%2590%25E3%2582%258B%25E3%2581%25BF%25E3%2582%25B7%25E3%2583%25AA%25E3%2583%25BC%25E3%2582%25BA%25E3%2580%258C%25E3%2581%2586%25E3%2582%258B%25E3%2581%25BD%25E3%2581%25A1%25E3%2582%2583%25E3%2581%25A1%25E3%2582%2583%25E3%2582%2593%25E3%2580%258D',
'https://shopdisney.disney.co.jp/goods/4550424793829.html?isProductSearch=0&plpPosition=6&guestFacing=%25E3%2581%258A%25E3%2581%2599%25E3%2581%2599%25E3%2582%2581%25E3%2582%25B0%25E3%2583%2583%25E3%2582%25BA%25E7%2589%25B9%25E9%259B%2586-%25E3%2581%25AC%25E3%2581%2584%25E3%2581%2590%25E3%2582%258B%25E3%2581%25BF%25E3%2582%25B7%25E3%2583%25AA%25E3%2583%25BC%25E3%2582%25BA%25E3%2580%258C%25E3%2581%2586%25E3%2582%258B%25E3%2581%25BD%25E3%2581%25A1%25E3%2582%2583%25E3%2581%25A1%25E3%2582%2583%25E3%2582%2593%25E3%2580%258D',
'https://shopdisney.disney.co.jp/goods/4550424862785.html?isProductSearch=0&plpPosition=8&guestFacing=%25E3%2581%258A%25E3%2581%2599%25E3%2581%2599%25E3%2582%2581%25E3%2582%25B0%25E3%2583%2583%25E3%2582%25BA%25E7%2589%25B9%25E9%259B%2586-%25E3%2581%25AC%25E3%2581%2584%25E3%2581%2590%25E3%2582%258B%25E3%2581%25BF%25E3%2582%25B7%25E3%2583%25AA%25E3%2583%25BC%25E3%2582%25BA%25E3%2580%258C%25E3%2581%2586%25E3%2582%258B%25E3%2581%25BD%25E3%2581%25A1%25E3%2582%2583%25E3%2581%25A1%25E3%2582%2583%25E3%2582%2593%25E3%2580%258D',
'https://shopdisney.disney.co.jp/goods/4550586628243.html?isProductSearch=0&plpPosition=9&guestFacing=%25E3%2581%258A%25E3%2581%2599%25E3%2581%2599%25E3%2582%2581%25E3%2582%25B0%25E3%2583%2583%25E3%2582%25BA%25E7%2589%25B9%25E9%259B%2586-%25E3%2581%25AC%25E3%2581%2584%25E3%2581%2590%25E3%2582%258B%25E3%2581%25BF%25E3%2582%25B7%25E3%2583%25AA%25E3%2583%25BC%25E3%2582%25BA%25E3%2580%258C%25E3%2581%2586%25E3%2582%258B%25E3%2581%25BD%25E3%2581%25A1%25E3%2582%2583%25E3%2581%25A1%25E3%2582%2583%25E3%2582%2593%25E3%2580%258D',
'https://shopdisney.disney.co.jp/goods/4550586628236.html?isProductSearch=0&plpPosition=10&guestFacing=%25E3%2581%258A%25E3%2581%2599%25E3%2581%2599%25E3%2582%2581%25E3%2582%25B0%25E3%2583%2583%25E3%2582%25BA%25E7%2589%25B9%25E9%259B%2586-%25E3%2581%25AC%25E3%2581%2584%25E3%2581%2590%25E3%2582%258B%25E3%2581%25BF%25E3%2582%25B7%25E3%2583%25AA%25E3%2583%25BC%25E3%2582%25BA%25E3%2580%258C%25E3%2581%2586%25E3%2582%258B%25E3%2581%25BD%25E3%2581%25A1%25E3%2582%2583%25E3%2581%25A1%25E3%2582%2583%25E3%2582%2593%25E3%2580%258D',
'https://shopdisney.disney.co.jp/goods/4550424550934.html?isProductSearch=0&plpPosition=11&guestFacing=%25E3%2581%258A%25E3%2581%2599%25E3%2581%2599%25E3%2582%2581%25E3%2582%25B0%25E3%2583%2583%25E3%2582%25BA%25E7%2589%25B9%25E9%259B%2586-%25E3%2581%25AC%25E3%2581%2584%25E3%2581%2590%25E3%2582%258B%25E3%2581%25BF%25E3%2582%25B7%25E3%2583%25AA%25E3%2583%25BC%25E3%2582%25BA%25E3%2580%258C%25E3%2581%2586%25E3%2582%258B%25E3%2581%25BD%25E3%2581%25A1%25E3%2582%2583%25E3%2581%25A1%25E3%2582%2583%25E3%2582%2593%25E3%2580%258D',
'https://shopdisney.disney.co.jp/goods/4550424551078.html?isProductSearch=0&plpPosition=12&guestFacing=%25E3%2581%258A%25E3%2581%2599%25E3%2581%2599%25E3%2582%2581%25E3%2582%25B0%25E3%2583%2583%25E3%2582%25BA%25E7%2589%25B9%25E9%259B%2586-%25E3%2581%25AC%25E3%2581%2584%25E3%2581%2590%25E3%2582%258B%25E3%2581%25BF%25E3%2582%25B7%25E3%2583%25AA%25E3%2583%25BC%25E3%2582%25BA%25E3%2580%258C%25E3%2581%2586%25E3%2582%258B%25E3%2581%25BD%25E3%2581%25A1%25E3%2582%2583%25E3%2581%25A1%25E3%2582%2583%25E3%2582%2593%25E3%2580%258D',
'https://meccha-japan.com/ja/pouches/121156-pouch-pokemon-wakka-de-ikka.html',
'https://item.rakuten.co.jp/wildleap02/b0cjtcf771/?scid=af_pc_etc&sc2id=af_113_0_10001868&icm_acid=255-776-8501&icm_cid=17549940929&icm_agid=&ifd=57&iasid=wem_icbs_&gclid=Cj0KCQiA7OqrBhD9ARIsAK3UXh1BF0uRsYKyOdh0wTVtha4beyXUZOtDgjihpmE6CMTucCcHGhkL_hQaAvr0EALw_wcB',
'https://store.shopping.yahoo.co.jp/sora3/6400000051998.html',
'https://jp.mercari.com/shops/product/fhkAwwTeRSnMhai8QASz8o',
'https://www.amazon.co.jp/dp/B09WMVHYD5',
'https://www.amazon.co.jp/dp/B089P7LBTP',
'https://store.jp.square-enix.com/item/MMC06011_2.html',
'https://www.amazon.co.jp/dp/B0BXBFF2P6',
'https://tsurumai-hobby.jp/item/4560308575922',
'https://ec.treasure-f.com/item/1053006538755606?gad_source=1&gclid=Cj0KCQiAj_CrBhD-ARIsAIiMxT_WuEqb8Xy4UR6xpB8ADTObGw15zCeNlQxp-P9vCgS6oU7sYQ5jlNIaAtscEALw_wcB',
'https://ec.treasure-f.com/item/1004016076270004',
'https://ec.treasure-f.com/item/1035008920493042',
'https://paypayfleamarket.yahoo.co.jp/item/z217975502',
'https://www.2ndstreet.jp/goods/detail/goodsId/2320343805536/shopsId/30700',
'https://i.lumine.jp/item/865230038050001',
'https://i.lumine.jp/item/865230038050001',
'https://i.lumine.jp/item/865230038050001',
'https://i.lumine.jp/item/865230038050001',
'https://i.lumine.jp/item/865230038050001',
'https://i.lumine.jp/item/865230038050001',
'https://i.lumine.jp/item/865230038050001',
'https://i.lumine.jp/item/865230038050001',
'https://i.lumine.jp/item/865230038050001',
'https://i.lumine.jp/item/865230038050001',
'https://i.lumine.jp/item/865230038050001',
'https://i.lumine.jp/item/865230038050001',
'https://i.lumine.jp/item/865230038050001',
'https://i.lumine.jp/item/865230038050001',
'https://netmall.hardoff.co.jp/product/3050861/',
'https://pqowmz.duallead.top/index.php?main_page=product_info&products_id=11432',
'https://www.ec.otakarasouko.com/shopdetail/000000457901/',
'https://www.amazon.co.jp/dp/B0B9ZN7F59',
'https://item.rakuten.co.jp/hugallhobby/bti0250135/',
'https://www.amazon.co.jp/dp/B0CCT2R3QH',
'https://store.shopping.yahoo.co.jp/squacy/k5s10-0001.html',
'https://www.suruga-ya.jp/product/detail/602262321',
'https://www.amazon.co.jp/dp/B0C1V47TMJ',
'https://www.kyd-store.jp/c/genre/revo/nr008',
'https://item.fril.jp/9595aa585bcb94f12e17036a35554664',
'https://fullahead.shop/index.php?main_page=product_info&products_id=42019383',
'https://www.amazon.co.jp/dp/B005JVNF84',
'https://www.amazon.co.jp/dp/B0CCMT4TJ3',
'https://www.suruga-ya.jp/product/detail/646137608',
'https://munyugurumi.jp/itemdetail?ItemID=149176',
'https://munyugurumi.jp/itemdetail?ItemID=149176',
'https://item.rakuten.co.jp/capital-kyoto/nakajima-167101-22/?scid=af_pc_etc&sc2id=af_113_0_10001868&icm_agid=&icm_cid=17549940929&ifd=57&gclid=Cj0KCQiAsvWrBhC0ARIsAO4E6f_oTxgVcDlWT92R9Vwnu_IhiyfPbHZEVwuG-xPWq48nbMW3Nj0n2AAaAlzkEALw_wcB&icm_acid=255-776-8501&iasid=wem_icbs_',
];
 
const data = [];
 
const csvWriter = createCsvWriter({
  path: 'output.csv',
  encoding: 'utf8',
  header: [
      { id: 'url', title: 'URL' },
      { id: 'price', title: 'Price' },
      { id: 'stock', title: 'Stock' }
  ]
});
 
// Axios header
const headers = {
  //'Accept-Language': 'en-US;q=0.9,en;q=0.8',
  'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
}
 
// 非同期処理を扱うためにasync functionを使用
async function fetchData() {
  for (const url of urls) {
    let price = '';
    let stock = '';
    try {
      const response = await axios.get(url, { headers, timeout: 20000 });
 
      const $ = cheerio.load(response.data);
 
      if (url.includes('www.amazon.co.jp')) {
        const priceElement = $('#centerCol').find('#corePriceDisplay_desktop_feature_div').find('span.a-price-whole');
        if (priceElement.length > 0) {
          price = priceElement.html().trim();
        } else {
          price = '';
        }
        const stockInputElement = $('div.a-button-stack').find('input');
        if (stockInputElement.length > 0 ) {
          // Value属性の値を取得
          const valueAttribute = stockInputElement.attr('value');
          if (valueAttribute === 'カートに入れる') {
            stock = 'あり';
          } else {
            stock = 'なし';
          }
        } else {
          stock = 'なし';
        }
      } else if (url.includes('shopdisney.disney.co.jp')) {
              // 価格を取得
        price = $('body > div.page > main > div:nth-child(9) > div.product-detail.product-detail__section-bound.product-wrapper.rating--unavailable > div.product-detail__inner-container > div:nth-child(1)')
        .find('div.price').find('span.value').attr('content');
 
        // 在庫状況を確認
        const addToCartBtn = $('body > div.page > main > div:nth-child(9) > div.product-detail.product-detail__section-bound.product-wrapper.rating--unavailable > div.product-detail__inner-container > div.col-12.col-sm-12.col-md-6.col-lg-4.col-xl-3.product-detail__content-summary.product-detail__content-summary--details.product-detail__content-summary--name > div > div.col-12.prices-add-to-cart-actions > div > div:nth-child(1) > button')
        .find('span.btn').children('span').text().trim();
        if (addToCartBtn === 'カートに入れる') {
          stock = 'あり';
        } else {
          stock = 'なし';
        }
      } else if (url.includes('item.rakuten.co.jp')) {
        price = $('#rakutenLimitedId_cart').find('#priceCalculationConfig').attr('data-price');
 
        const addToCartBtn = $('#AddToCartPurchaseButtonFixed').find('button[aria-label="カートに追加"]');
        const isDisabled = addToCartBtn.prop('disabled') !== undefined || addToCartBtn.attr('disabled') !== undefined;
        if (isDisabled) {
          stock = 'なし';
        } else {
          stock = 'あり';
        }  
      } else if (url.includes('www.suruga-ya.jp/product/detail')){
        const priceGroupEl = $('body > div.dialog-off-canvas-main-canvas > div.container_suru.padB40 > div:nth-child(9) > div.col-8.padL32 > div.d-flex.justify-content-start > div.w-70.pr-5 > div.price_group.mb-3');
        const priceWithoutCamma = priceGroupEl.find('span.text-price-detail.price-buy').text().replace(/,/g, '');
        price = parseInt(priceWithoutCamma, 10);
        stock = 'あり'
      }
      else if (url.includes('ec.treasure-f.com/item')) {
        const itemElRight = $('body > main > div.main > div.clearfix > div.item-detail-area-right');
        const itempriceArea = itemElRight.find('.item-detail-area-right-price');
        const priceCurrency = itempriceArea.children('p').find('.disp-tax-in').text().replace(/[^\d]/g, '');
        price = parseInt(priceCurrency, 10);
        console.log(price);
 
        const addToCartBtnEl = itemElRight.find('#item-submitform').find('#item-cartbutton');
        if (addToCartBtnEl.length > 0) {
          stock = 'あり';
        } else {
          stock = 'なし';
        }
      } else if (url.includes('item.fril.jp')) {
        const price = $('body > div.drawer-overlay > div.container.new-rakuma > div > div.col-lg-12.col-md-12.col-sm-12.col-xs-12 > div.row > div > article > div > div.col-lg-5.col-md-5.col-sm-12.col-xs-12.right-section > section > div.item-info__header > div:nth-child(2) > div > p > span.item__price')
          .text().replace(/[^\d]/g, '');
 
        const sellBtnEl = $('body > div.drawer-overlay > div.container.new-rakuma > div > div.col-lg-12.col-md-12.col-sm-12.col-xs-12 > div.row > div > article > div > div.col-lg-5.col-md-5.col-sm-12.col-xs-12.right-section > section > div:nth-child(4) > div > div.btn-buy-fixed.clearfix > p.sell-btn-fixed');
        if (sellBtnEl.length > 0) {
          stock = 'あり';
        } else {
          stock = 'なし';
        }
      } else {
        price = '';
        stock = '';
      }
 
      data.push({ url, price, stock });
      console.log(data);
 
    } catch (error) {
      console.error('エラー:', error);
      data.push({ url, price, stock });
    }
  }
 
 // 全ての非同期処理が完了した後にCSVに書き込み
  await csvWriter.writeRecords(data);
  console.log('CSVにデータが書き込まれました。');
}
 
// fetchDataを呼び出し
fetchData();
nodejs/scraping.1702899975.txt.gz · 最終更新: 2023/12/18 20:46 by mikoto