ber*_*nie 117
import csv, sqlite3
con = sqlite3.connect(":memory:")
cur = con.cursor()
cur.execute("CREATE TABLE t (col1, col2);") # use your column names here
with open('data.csv','rb') as fin: # `with` statement available in 2.5+
# csv.DictReader uses first line in file for column headings by default
dr = csv.DictReader(fin) # comma is default delimiter
to_db = [(i['col1'], i['col2']) for i in dr]
cur.executemany("INSERT INTO t (col1, col2) VALUES (?, ?);", to_db)
con.commit()
con.close()
Ten*_*urg 75
创建一个sqlite连接到磁盘上的文件是留给读者的练习...但现在有一个双线程由熊猫库实现
df = pandas.read_csv(csvfile)
df.to_sql(table_name, conn, if_exists='append', index=False)
Jak*_*aas 16
你是对的，这.import是要走的路，但这是来自 SQLite3 命令行程序的命令。这个问题的很多顶级答案都涉及原生 python 循环，但如果你的文件很大（我的是 10^6 到 10^7 条记录），你想避免将所有内容读入 Pandas 或使用原生 python 列表理解/循环（虽然我没有为它们计时）。
对于大文件，我认为最好的选择是使用subprocess.run()执行sqlite的导入命令。在下面的示例中，我假设该表已经存在，但 csv 文件的第一行有标题。有关更多信息，请参阅.import文档。
subprocess.run()from pathlib import Path
db_name = Path('my.db').resolve()
csv_file = Path('file.csv').resolve()
result = subprocess.run(['sqlite3',
str(db_name),
'-cmd',
'.mode csv',
'.import --skip 1 ' + str(csv_file).replace('\\','\\\\')
+' <table_name>'],
capture_output=True)
编辑说明：sqlite3 的.import命令已得到改进，因此它可以将第一行视为标题名称，甚至可以跳过前x行（需要版本 >=3.32，如本答案所述。如果您有旧版本的 sqlite3，您可能需要首先创建表，然后在导入前去掉csv的第一行。--skip 1参数会在3.32之前报错
说明
在命令行中，您要查找的命令是sqlite3 my.db -cmd ".mode csv" ".import file.csv table". subprocess.run()运行命令行进程。to 的参数subprocess.run()是一个字符串序列，它被解释为一个命令，后跟它的所有参数。
sqlite3 my.db 打开数据库-cmd数据库后的标志允许您将多个后续命令传递给 sqlite 程序。在 shell 中，每个命令都必须用引号引起来，但在这里，它们只需要是序列中自己的元素'.mode csv' 做你所期望的'.import --skip 1'+str(csv_file).replace('\\','\\\\')+' <table_name>'是导入命令。-cmd作为带引号的字符串传递，如果您有 Windows 目录路径，则需要将反斜杠加倍。不是问题的重点，但这是我使用的。同样，我不想在任何时候将整个文件读入内存：
with open(csv, "r") as source:
source.readline()
with open(str(csv)+"_nohead", "w") as target:
shutil.copyfileobj(source, target)
Mar*_*tos 12
该.import命令是sqlite3命令行工具的一项功能.要在Python中执行此操作,您只需使用Python具有的任何功能(例如csv模块)加载数据,并按常规方式插入数据.
这样,您还可以控制插入的类型,而不是依赖于sqlite3看似无证的行为.
Guy*_*y L 12
我的2美分(更通用):
import csv, sqlite3
import logging
def _get_col_datatypes(fin):
dr = csv.DictReader(fin) # comma is default delimiter
fieldTypes = {}
for entry in dr:
feildslLeft = [f for f in dr.fieldnames if f not in fieldTypes.keys()]
if not feildslLeft: break # We're done
for field in feildslLeft:
data = entry[field]
# Need data to decide
if len(data) == 0:
continue
if data.isdigit():
fieldTypes[field] = "INTEGER"
else:
fieldTypes[field] = "TEXT"
# TODO: Currently there's no support for DATE in sqllite
if len(feildslLeft) > 0:
raise Exception("Failed to find all the columns data types - Maybe some are empty?")
return fieldTypes
def escapingGenerator(f):
for line in f:
yield line.encode("ascii", "xmlcharrefreplace").decode("ascii")
def csvToDb(csvFile, outputToFile = False):
# TODO: implement output to file
with open(csvFile,mode='r', encoding="ISO-8859-1") as fin:
dt = _get_col_datatypes(fin)
fin.seek(0)
reader = csv.DictReader(fin)
# Keep the order of the columns name just as in the CSV
fields = reader.fieldnames
cols = []
# Set field and type
for f in fields:
cols.append("%s %s" % (f, dt[f]))
# Generate create table statement:
stmt = "CREATE TABLE ads (%s)" % ",".join(cols)
con = sqlite3.connect(":memory:")
cur = con.cursor()
cur.execute(stmt)
fin.seek(0)
reader = csv.reader(escapingGenerator(fin))
# Generate insert statement:
stmt = "INSERT INTO ads VALUES(%s);" % ','.join('?' * len(cols))
cur.executemany(stmt, reader)
con.commit()
return con
非常感谢bernie的回答!不得不稍微调整一下 - 这对我有用:
import csv, sqlite3
conn = sqlite3.connect("pcfc.sl3")
curs = conn.cursor()
curs.execute("CREATE TABLE PCFC (id INTEGER PRIMARY KEY, type INTEGER, term TEXT, definition TEXT);")
reader = csv.reader(open('PC.txt', 'r'), delimiter='|')
for row in reader:
to_db = [unicode(row[0], "utf8"), unicode(row[1], "utf8"), unicode(row[2], "utf8")]
curs.execute("INSERT INTO PCFC (type, term, definition) VALUES (?, ?, ?);", to_db)
conn.commit()
我的文本文件(PC.txt)如下所示:
1 | Term 1 | Definition 1
2 | Term 2 | Definition 2
3 | Term 3 | Definition 3
小智 7
#!/usr/bin/python
# -*- coding: utf-8 -*-
import sys, csv, sqlite3
def main():
con = sqlite3.connect(sys.argv[1]) # database file input
cur = con.cursor()
cur.executescript("""
DROP TABLE IF EXISTS t;
CREATE TABLE t (COL1 TEXT, COL2 TEXT);
""") # checks to see if table exists and makes a fresh table.
with open(sys.argv[2], "rb") as f: # CSV file input
reader = csv.reader(f, delimiter=',') # no header information with delimiter
for row in reader:
to_db = [unicode(row[0], "utf8"), unicode(row[1], "utf8")] # Appends data from CSV file representing and handling of text
cur.execute("INSERT INTO neto (COL1, COL2) VALUES(?, ?);", to_db)
con.commit()
con.close() # closes connection to database
if __name__=='__main__':
main()
如果您的 CSV 文件非常大，以下是可行的解决方案。按照另一个答案的建议使用to_sql，但设置块大小，这样它就不会尝试立即处理整个文件。
import sqlite3
import pandas as pd
conn = sqlite3.connect('my_data.db')
c = conn.cursor()
users = pd.read_csv('users.csv')
users.to_sql('users', conn, if_exists='append', index = False, chunksize = 10000)
您还可以使用 Dask，如此处所述并行编写大量 Pandas DataFrame：
dto_sql = dask.delayed(pd.DataFrame.to_sql)
out = [dto_sql(d, 'table_name', db_url, if_exists='append', index=True)
for d in ddf.to_delayed()]
dask.compute(*out)
请参阅此处了解更多详细信息。
"""
cd Final_Codes
python csv_to_db.py
CSV to SQL DB
"""
import csv
import sqlite3
import os
import fnmatch
UP_FOLDER = os.path.dirname(os.getcwd())
DATABASE_FOLDER = os.path.join(UP_FOLDER, "Databases")
DBNAME = "allCompanies_database.db"
def getBaseNameNoExt(givenPath):
"""Returns the basename of the file without the extension"""
filename = os.path.splitext(os.path.basename(givenPath))[0]
return filename
def find(pattern, path):
"""Utility to find files wrt a regex search"""
result = []
for root, dirs, files in os.walk(path):
for name in files:
if fnmatch.fnmatch(name, pattern):
result.append(os.path.join(root, name))
return result
if __name__ == "__main__":
Database_Path = os.path.join(DATABASE_FOLDER, DBNAME)
# change to 'sqlite:///your_filename.db'
csv_files = find('*.csv', DATABASE_FOLDER)
con = sqlite3.connect(Database_Path)
cur = con.cursor()
for each in csv_files:
with open(each, 'r') as fin: # `with` statement available in 2.5+
# csv.DictReader uses first line in file for column headings by default
dr = csv.DictReader(fin) # comma is default delimiter
TABLE_NAME = getBaseNameNoExt(each)
Cols = dr.fieldnames
numCols = len(Cols)
"""
for i in dr:
print(i.values())
"""
to_db = [tuple(i.values()) for i in dr]
print(TABLE_NAME)
# use your column names here
ColString = ','.join(Cols)
QuestionMarks = ["?"] * numCols
ToAdd = ','.join(QuestionMarks)
cur.execute(f"CREATE TABLE {TABLE_NAME} ({ColString});")
cur.executemany(
f"INSERT INTO {TABLE_NAME} ({ColString}) VALUES ({ToAdd});", to_db)
con.commit()
con.close()
print("Execution Complete!")
当您的文件夹中有很多 csv 文件并且您希望立即将其转换为单个 .db 文件时，这应该会派上用场！
请注意，您不必事先知道文件名、表名或字段名（列名）！
如果 CSV 文件必须作为 python 程序的一部分导入，那么为了简单和高效，您可以os.system按照以下建议的方式使用：
import os
cmd = """sqlite3 database.db <<< ".import input.csv mytable" """
rc = os.system(cmd)
print(rc)
要点是，通过指定数据库的文件名，假设读取没有错误，数据将自动保存。