管道模塊
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymysql
class MyspiderPipeline(object):
def __int__(self):
print('my pipelines is ready....__init__ invoking....')
def open_spider(self,spider):
'''
? ? ? ? 建立數(shù)據(jù)庫(kù)連接
? ? ? ? :paramspider:
? ? ? ? :return:
'''
? ? ? ? self.my_conn = pymysql.connect(
host ="localhost",
port =3306,
database ="jobs",
user ='root',
password ='123',
charset ='utf8'
? ? ? ? )
#獲取游標(biāo)對(duì)象
? ? ? ? self.my_cursor =self.my_conn.cursor()
def process_item(self,item,spider):
print('----------------------process_item is invoking----------------------')
# print item['name']
# print item['salary']
# print item['company']
# print item['day']
# print item['experience']
# print item['area']
# print item['number']
# print item['nature']
# print item['education']
# print item['description']
? ? ? ? insert_sql ="insert into wuxi(name, salary, company, day,experience,area,number,nature,education,description) values(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
? ? ? ? self.my_cursor.execute(insert_sql,[item['name'],item['salary'],item['company'],item['day'],item['experience'],item['area'],item['number'],item['nature'],item['education'],item['description']])
def close_spider(self, spider):
#提交MySQL語(yǔ)句
? ? ? ? self.my_conn.commit()
#關(guān)閉游標(biāo)對(duì)象與數(shù)據(jù)庫(kù)連接
? ? ? ? self.my_cursor.close()
self.my_conn.close()
items
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class MyspiderItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
? ? name = scrapy.Field()
salary = scrapy.Field()
company = scrapy.Field()
day = scrapy.Field()
experience = scrapy.Field()
area = scrapy.Field()
number = scrapy.Field()
nature = scrapy.Field()
education = scrapy.Field()
description = scrapy.Field()
