pip install scrapy

# 新建项目 infini_spiders
scrapy startproject infini_spiders

# 初始化爬虫
cd infini_spiders/spiders
scrapy genspider blog infinilabs.cn

from typing import Any, Iterable
import scrapy
from bs4 import BeautifulSoup
from scrapy.http import Response

class BlogSpider(scrapy.Spider):
    name = "blog"
    allowed_domains = ["infinilabs.cn"]
    start_urls = ["https://infinilabs.cn/blog/"]

    def parse(self, response):
        links = response.css("div.blogs a")
        yield from response.follow_all(links, self.parse_blog)

    def parse_blog(self, response):
        title = response.xpath('//div[@class="title"]/text()').extract_first()
        tags = response.xpath('//div[@class="tags"]/div[@class="tag"]/text()').extract()
        url = response.url
        author = response.xpath('//div[@class="logo"]/div[@class="name"]//text()').extract_first()
        date = response.xpath('//div[@class="date"]/text()').extract_first()
        all_text = response.xpath('//p//text() | //h3/text() | //h2/text() | //h4/text() | //ol/li//text()').extract()
        content = '\n'.join(all_text)

        yield {
            'title': title,
            'tags': tags,
            'url': url,
            'author': author,
            'date': date,
            'content': content
        }

pip install ScrapyElasticSearch

ITEM_PIPELINES = {
    'scrapyelasticsearch.scrapyelasticsearch.ElasticSearchPipeline': 10
}

ELASTICSEARCH_SERVERS = ['http://192.168.56.3:9210']
ELASTICSEARCH_INDEX = 'scrapy'
ELASTICSEARCH_INDEX_DATE_FORMAT = '%Y-%m-%d'
ELASTICSEARCH_TYPE = '_doc'
ELASTICSEARCH_USERNAME = 'admin'
ELASTICSEARCH_PASSWORD = '9423d1d5345ed6d0db19'

scrapy crawl blog