sqlite3 scrapy

[tokauf at gmail.com]

Hi,

I have a little scrapy-script: scrapy gets the links from a webpage; this works fine. Then I want to write these links in a sqlite3-table. There is no error-note. But in the database are not any records. What is the problem here? My code:

 # -*- coding: utf-8 -*-
  2 
  3 import scrapy
  4 import sqlite3
  5 
  6 class MySpider(scrapy.Spider):
  7
  8     name            = "frisch2"
  9     allowed_domains = ["frischblau.de"]
 10     start_urls      = ["http://www.frischblau.de/",]
 11 
 12     def parse(self, response):
 13         
 14         for href in response.xpath("//ul/li/a/@href"):
 15             
 16             url = response.urljoin(href.extract())
 17             yield scrapy.Request(url, callback=self.parse_dir_contents)
 18             
 19     def parse_dir_contents(self, response):
 20         
 21         sql  = u' '
 22         conn = sqlite3.connect('frisch.db')
 23         cur  = conn.cursor()
 24         counter = 3;
 25         
 26         try:
 27             cur.execute("SELECT SQLITE_VERSION()")
 28             print "-> SQLite version: %s" % cur.fetchone()
 29             
 30             item = response.xpath('//a/@href').extract()
 31             for el in item:
 32             
 33                 #print("----------> ", type(el))
 34                 sql = "INSERT INTO Frisch VALUES(" + unicode(counter) + ", " + "'" + el.strip() + "');"
 35             
 36                 print (sql)
 37                 cur.execute(sql)
 38                 conn.commit
 39                 counter = int(counter)
 40                 counter += 1
 41 
 42         except sqlite3.Error as e:
 43             print "Error %s:" % e.args[0]
 44             conn.close()
 45 
                                                                                                                                                                                                                                           

Thanks for help.

o-o

Thomas