sqlite3 - Python scrapy pipeline error -
i'm developing spider scraping pages , have issues scrapy-pipeline.. don't know why.. what's wrong in script?
# -*- coding: utf-8 -*- # define item pipelines here # # don't forget add pipeline item_pipelines setting # see: http://doc.scrapy.org/en/latest/topics/item-pipeline.html scrapy.exceptions import dropitem import sqlite3 con = none class realtybasepipeline(object): def __init__(self): self.setupdbcon() self.createtables() def process_item(self, item, spider): self.storeindb(item) return item def storeindb(self, item): dealerid = self.cur.lastrowid self.storesrealityinfoindb(item) self.storedealerinfoindb(item) def storesrealityinfoindb(self, item): self.cur.execute("insert sreality(\ name, \ price, \ url, \ estatetype, \ adress, \ createdate, \ source, \ dealername, \ dealermail, \ dealerphoto, \ ) \ values( ?, ?, ?, ?, ?, ?, ?, ?, ? )", \ ( \ item.get('name'), item.get('price'), item.get('url'), item.get('estatetype'), item.get('adress'), item.get('createdate'), item.get('source'), item.get('dealername'), item.get('dealermail'), item.get('dealerphoto'), )) self.con.commit() def storedealerinfoindb(self, item): self.cur.execute("insert actors(\ dealername, \ dealermail, \ dealerphoto \ ) \ values(?,?,?)", ( item.get('dealername'), item.get('dealermail'), item.get('dealerphoto'), )) self.con.commit() def setupdbcon(self): self.con = sqlite3.connect('test.db') self.cur = self.con.cursor() # class destructor. called automaticly python's garbage collecter once class no longer used. def __del__(self): self.closedb() # i'm droping tables if exist before run script each time, # don't duplicate info. def createtables(self): #self.dropsrealitytable() #self.dropdealerstable() self.createsrealitytable() self.createdealerstable() def createsrealitytable(self): self.cur.execute("create table if not exists sreality(name text primary key not null, \ price text, \ url text, \ photo text, \ estatetype text, \ adress text, \ createdate text, \ source text, \ dealername text, \ dealermail text, \ dealerphoto text, \ )") def createdealerstable(self): self.cur.execute("create table if not exists actors(dealername text primary key not null, \ dealermail text, \ dealerphoto text )") def dropsrealitytable(self): self.cur.execute("drop table if exists sreality") def dropdealerstable(self): self.cur.execute("drop table if exists dealers") def closedb(self): self.con.close() after run spider, throws me errors..
file "/home/pr0n/dropbox/realtybase/realtybase/pipelines.py", line 16, in __init__ self.createtables() file "/home/pr0n/dropbox/realtybase/realtybase/pipelines.py", line 83, in createtables self.createsrealitytable() file "/home/pr0n/dropbox/realtybase/realtybase/pipelines.py", line 99, in createsrealitytable )") sqlite3.operationalerror: near ")": syntax error
there extra comma before closing parenthesis:
self.cur.execute("create table if not exists sreality(name text primary key not null, \ price text, \ url text, \ photo text, \ estatetype text, \ adress text, \ createdate text, \ source text, \ dealername text, \ dealermail text, \ dealerphoto text, \ # < here )") as side point, improve readability, can write multi-line query in triple quotes:
self.cur.execute(""" create table if not exists sreality (name text primary key not null, price text, url text, photo text, estatetype text, adress text, createdate text, source text, dealername text, dealermail text, dealerphoto text) """) you may consider abstracting out orm, sqlalchemy.