diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -877,19 +877,17 @@ timestamp converter. .. _sqlite3-controlling-transactions: Controlling Transactions ------------------------ By default, the :mod:`sqlite3` module opens transactions implicitly before a Data Modification Language (DML) statement (i.e. -``INSERT``/``UPDATE``/``DELETE``/``REPLACE``), and commits transactions -implicitly before a non-DML, non-query statement (i. e. -anything other than ``SELECT`` or the aforementioned). +``INSERT``/``UPDATE``/``DELETE``/``REPLACE``). So if you are within a transaction and issue a command like ``CREATE TABLE ...``, ``VACUUM``, ``PRAGMA``, the :mod:`sqlite3` module will commit implicitly before executing that command. There are two reasons for doing that. The first is that some of these commands don't work within transactions. The other reason is that sqlite3 needs to keep track of the transaction state (if a transaction is active or not). The current transaction state is exposed through the :attr:`Connection.in_transaction` attribute of the connection object. @@ -899,16 +897,19 @@ You can control which kind of ``BEGIN`` call, or via the :attr:`isolation_level` property of connections. If you want **autocommit mode**, then set :attr:`isolation_level` to None. Otherwise leave it at its default, which will result in a plain "BEGIN" statement, or set it to one of SQLite's supported isolation levels: "DEFERRED", "IMMEDIATE" or "EXCLUSIVE". +.. versionchanged:: 3.6 + :mod:`sqlite3` used to implicitly commit an open transaction before DDL + statements. This is no longer the case. Using :mod:`sqlite3` efficiently -------------------------------- Using shortcut methods ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -480,13 +480,16 @@ Changes in the Python API an exception like :exc:`SystemExit` or :exc:`KeyboardInterrupt`, :meth:`~socketserver.BaseServer.handle_error` is no longer called, and the exception will stop a single-threaded server. (Contributed by Martin Panter in :issue:`23430`.) * :func:`spwd.getspnam` now raises a :exc:`PermissionError` instead of :exc:`KeyError` if the user doesn't have privileges. +* :mod:`sqlite3` no longer implicitly commit an open transaction before DDL + statements. + Changes in the C API -------------------- * :c:func:`Py_Exit` (and the main interpreter) now override the exit status with 120 if flushing buffered data failed. See :issue:`5319`. diff --git a/Lib/sqlite3/test/transactions.py b/Lib/sqlite3/test/transactions.py --- a/Lib/sqlite3/test/transactions.py +++ b/Lib/sqlite3/test/transactions.py @@ -47,24 +47,16 @@ class TransactionTests(unittest.TestCase self.cur2.close() self.con2.close() try: os.unlink(get_db_path()) except OSError: pass - def CheckDMLdoesAutoCommitBefore(self): - self.cur1.execute("create table test(i)") - self.cur1.execute("insert into test(i) values (5)") - self.cur1.execute("create table test2(j)") - self.cur2.execute("select i from test") - res = self.cur2.fetchall() - self.assertEqual(len(res), 1) - def CheckInsertStartsTransaction(self): self.cur1.execute("create table test(i)") self.cur1.execute("insert into test(i) values (5)") self.cur2.execute("select i from test") res = self.cur2.fetchall() self.assertEqual(len(res), 0) def CheckUpdateStartsTransaction(self): @@ -167,38 +159,60 @@ class TransactionTests(unittest.TestCase except: self.fail("InterfaceError should have been raised") class SpecialCommandTests(unittest.TestCase): def setUp(self): self.con = sqlite.connect(":memory:") self.cur = self.con.cursor() - def CheckVacuum(self): - self.cur.execute("create table test(i)") - self.cur.execute("insert into test(i) values (5)") - self.cur.execute("vacuum") - def CheckDropTable(self): self.cur.execute("create table test(i)") self.cur.execute("insert into test(i) values (5)") self.cur.execute("drop table test") def CheckPragma(self): self.cur.execute("create table test(i)") self.cur.execute("insert into test(i) values (5)") self.cur.execute("pragma count_changes=1") def tearDown(self): self.cur.close() self.con.close() +class TransactionalDDL(unittest.TestCase): + def setUp(self): + self.con = sqlite.connect(":memory:") + + def CheckDdlDoesNotAutostartTransaction(self): + """ + For backwards compatibility reasons, DDL statements should not + implicitly start a transaction. + """ + self.con.execute("create table test(i)") + self.con.rollback() + self.con.execute("select * from test") + + def CheckTransactionalDDL(self): + """ + You can achieve transactional DDL by issuing a BEGIN statement manually. + """ + self.con.execute("begin") + self.con.execute("create table test(i)") + self.con.rollback() + with self.assertRaises(sqlite.OperationalError): + self.con.execute("select * from test") + + def tearDown(self): + self.con.close() + def suite(): default_suite = unittest.makeSuite(TransactionTests, "Check") special_command_suite = unittest.makeSuite(SpecialCommandTests, "Check") - return unittest.TestSuite((default_suite, special_command_suite)) + ddl_suite = unittest.makeSuite(TransactionalDDL, "Check") + return unittest.TestSuite((default_suite, special_command_suite, ddl_suite)) def test(): runner = unittest.TextTestRunner() runner.run(suite()) if __name__ == "__main__": test() diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -24,54 +24,16 @@ #include "cursor.h" #include "module.h" #include "util.h" PyObject* pysqlite_cursor_iternext(pysqlite_Cursor* self); static const char errmsg_fetch_across_rollback[] = "Cursor needed to be reset because of commit/rollback and can no longer be fetched from."; -static pysqlite_StatementKind detect_statement_type(const char* statement) -{ - char buf[20]; - const char* src; - char* dst; - - src = statement; - /* skip over whitepace */ - while (*src == '\r' || *src == '\n' || *src == ' ' || *src == '\t') { - src++; - } - - if (*src == 0) - return STATEMENT_INVALID; - - dst = buf; - *dst = 0; - while (Py_ISALPHA(*src) && (dst - buf) < ((Py_ssize_t)sizeof(buf) - 2)) { - *dst++ = Py_TOLOWER(*src++); - } - - *dst = 0; - - if (!strcmp(buf, "select")) { - return STATEMENT_SELECT; - } else if (!strcmp(buf, "insert")) { - return STATEMENT_INSERT; - } else if (!strcmp(buf, "update")) { - return STATEMENT_UPDATE; - } else if (!strcmp(buf, "delete")) { - return STATEMENT_DELETE; - } else if (!strcmp(buf, "replace")) { - return STATEMENT_REPLACE; - } else { - return STATEMENT_OTHER; - } -} - static int pysqlite_cursor_init(pysqlite_Cursor* self, PyObject* args, PyObject* kwargs) { pysqlite_Connection* connection; if (!PyArg_ParseTuple(args, "O!", &pysqlite_ConnectionType, &connection)) { return -1; } @@ -423,19 +385,19 @@ PyObject* _pysqlite_query_execute(pysqli PyObject* parameters_list = NULL; PyObject* parameters_iter = NULL; PyObject* parameters = NULL; int i; int rc; PyObject* func_args; PyObject* result; int numcols; - int statement_type; PyObject* descriptor; PyObject* second_argument = NULL; + sqlite_int64 lastrowid; if (!check_cursor(self)) { goto error; } self->locked = 1; self->reset = 0; @@ -506,17 +468,17 @@ PyObject* _pysqlite_query_execute(pysqli operation_cstr = _PyUnicode_AsStringAndSize(operation, &operation_len); if (operation_cstr == NULL) goto error; /* reset description and rowcount */ Py_INCREF(Py_None); Py_SETREF(self->description, Py_None); - self->rowcount = -1L; + self->rowcount = 0L; func_args = PyTuple_New(1); if (!func_args) { goto error; } Py_INCREF(operation); if (PyTuple_SetItem(func_args, 0, operation) != 0) { goto error; @@ -545,53 +507,29 @@ PyObject* _pysqlite_query_execute(pysqli Py_CLEAR(self->statement); goto error; } } pysqlite_statement_reset(self->statement); pysqlite_statement_mark_dirty(self->statement); - statement_type = detect_statement_type(operation_cstr); - if (self->connection->begin_statement) { - switch (statement_type) { - case STATEMENT_UPDATE: - case STATEMENT_DELETE: - case STATEMENT_INSERT: - case STATEMENT_REPLACE: - if (!self->connection->inTransaction) { - result = _pysqlite_connection_begin(self->connection); - if (!result) { - goto error; - } - Py_DECREF(result); - } - break; - case STATEMENT_OTHER: - /* it's a DDL statement or something similar - - we better COMMIT first so it works for all cases */ - if (self->connection->inTransaction) { - result = pysqlite_connection_commit(self->connection, NULL); - if (!result) { - goto error; - } - Py_DECREF(result); - } - break; - case STATEMENT_SELECT: - if (multiple) { - PyErr_SetString(pysqlite_ProgrammingError, - "You cannot execute SELECT statements in executemany()."); - goto error; - } - break; + /* for backwards compatibility, do not start a transaction if a + DDL statement is encountered. if anybody wants transactional DDL, + they can issue a BEGIN statement manually. */ + if (self->connection->begin_statement && !sqlite3_stmt_readonly(self->statement->st) && !self->statement->is_ddl) { + if (sqlite3_get_autocommit(self->connection->db)) { + result = _pysqlite_connection_begin(self->connection); + if (!result) { + goto error; + } + Py_DECREF(result); } } - while (1) { parameters = PyIter_Next(parameters_iter); if (!parameters) { break; } pysqlite_statement_mark_dirty(self->statement); @@ -641,17 +579,17 @@ PyObject* _pysqlite_query_execute(pysqli } } if (pysqlite_build_row_cast_map(self) != 0) { PyErr_SetString(pysqlite_OperationalError, "Error while building row_cast_map"); goto error; } - if (rc == SQLITE_ROW || (rc == SQLITE_DONE && statement_type == STATEMENT_SELECT)) { + if (rc == SQLITE_ROW || rc == SQLITE_DONE) { if (self->description == Py_None) { Py_BEGIN_ALLOW_THREADS numcols = sqlite3_column_count(self->statement->st); Py_END_ALLOW_THREADS Py_SETREF(self->description, PyTuple_New(numcols)); if (!self->description) { goto error; @@ -668,53 +606,44 @@ PyObject* _pysqlite_query_execute(pysqli Py_INCREF(Py_None); PyTuple_SetItem(descriptor, 4, Py_None); Py_INCREF(Py_None); PyTuple_SetItem(descriptor, 5, Py_None); Py_INCREF(Py_None); PyTuple_SetItem(descriptor, 6, Py_None); PyTuple_SetItem(self->description, i, descriptor); } } } + if (!sqlite3_stmt_readonly(self->statement->st)) { + self->rowcount += (long)sqlite3_changes(self->connection->db); + } else { + self->rowcount= -1L; + } + + if (!multiple) { + Py_DECREF(self->lastrowid); + Py_BEGIN_ALLOW_THREADS + lastrowid = sqlite3_last_insert_rowid(self->connection->db); + Py_END_ALLOW_THREADS + self->lastrowid = _pysqlite_long_from_int64(lastrowid); + } + if (rc == SQLITE_ROW) { if (multiple) { PyErr_SetString(pysqlite_ProgrammingError, "executemany() can only execute DML statements."); goto error; } self->next_row = _pysqlite_fetch_one_row(self); if (self->next_row == NULL) goto error; } else if (rc == SQLITE_DONE && !multiple) { pysqlite_statement_reset(self->statement); Py_CLEAR(self->statement); } - switch (statement_type) { - case STATEMENT_UPDATE: - case STATEMENT_DELETE: - case STATEMENT_INSERT: - case STATEMENT_REPLACE: - if (self->rowcount == -1L) { - self->rowcount = 0L; - } - self->rowcount += (long)sqlite3_changes(self->connection->db); - } - - Py_DECREF(self->lastrowid); - if (!multiple && statement_type == STATEMENT_INSERT) { - sqlite_int64 lastrowid; - Py_BEGIN_ALLOW_THREADS - lastrowid = sqlite3_last_insert_rowid(self->connection->db); - Py_END_ALLOW_THREADS - self->lastrowid = _pysqlite_long_from_int64(lastrowid); - } else { - Py_INCREF(Py_None); - self->lastrowid = Py_None; - } - if (multiple) { pysqlite_statement_reset(self->statement); } Py_XDECREF(parameters); } error: /* just to be sure (implicit ROLLBACKs with ON CONFLICT ROLLBACK/OR diff --git a/Modules/_sqlite/cursor.h b/Modules/_sqlite/cursor.h --- a/Modules/_sqlite/cursor.h +++ b/Modules/_sqlite/cursor.h @@ -46,22 +46,16 @@ typedef struct int initialized; /* the next row to be returned, NULL if no next row available */ PyObject* next_row; PyObject* in_weakreflist; /* List of weak references */ } pysqlite_Cursor; -typedef enum { - STATEMENT_INVALID, STATEMENT_INSERT, STATEMENT_DELETE, - STATEMENT_UPDATE, STATEMENT_REPLACE, STATEMENT_SELECT, - STATEMENT_OTHER -} pysqlite_StatementKind; - extern PyTypeObject pysqlite_CursorType; PyObject* pysqlite_cursor_execute(pysqlite_Cursor* self, PyObject* args); PyObject* pysqlite_cursor_executemany(pysqlite_Cursor* self, PyObject* args); PyObject* pysqlite_cursor_getiter(pysqlite_Cursor *self); PyObject* pysqlite_cursor_iternext(pysqlite_Cursor *self); PyObject* pysqlite_cursor_fetchone(pysqlite_Cursor* self, PyObject* args); PyObject* pysqlite_cursor_fetchmany(pysqlite_Cursor* self, PyObject* args, PyObject* kwargs); diff --git a/Modules/_sqlite/statement.c b/Modules/_sqlite/statement.c --- a/Modules/_sqlite/statement.c +++ b/Modules/_sqlite/statement.c @@ -49,16 +49,17 @@ typedef enum { } parameter_type; int pysqlite_statement_create(pysqlite_Statement* self, pysqlite_Connection* connection, PyObject* sql) { const char* tail; int rc; const char* sql_cstr; Py_ssize_t sql_cstr_len; + const char* p; self->st = NULL; self->in_use = 0; sql_cstr = _PyUnicode_AsStringAndSize(sql, &sql_cstr_len); if (sql_cstr == NULL) { rc = PYSQLITE_SQL_WRONG_TYPE; return rc; @@ -67,16 +68,33 @@ int pysqlite_statement_create(pysqlite_S PyErr_SetString(PyExc_ValueError, "the query contains a null character"); return PYSQLITE_SQL_WRONG_TYPE; } self->in_weakreflist = NULL; Py_INCREF(sql); self->sql = sql; + /* determine if the statement is a DDL statement */ + self->is_ddl = 0; + for (p = sql_cstr; *p != 0; p++) { + switch (*p) { + case ' ': + case '\r': + case '\n': + case '\t': + continue; + } + + self->is_ddl = (PyOS_strnicmp(p, "create", 6) == 0) + || (PyOS_strnicmp(p, "drop", 4) == 0) + || (PyOS_strnicmp(p, "reindex", 7) == 0); + break; + } + Py_BEGIN_ALLOW_THREADS rc = sqlite3_prepare(connection->db, sql_cstr, -1, &self->st, &tail); Py_END_ALLOW_THREADS diff --git a/Modules/_sqlite/statement.h b/Modules/_sqlite/statement.h --- a/Modules/_sqlite/statement.h +++ b/Modules/_sqlite/statement.h @@ -33,16 +33,17 @@ typedef struct { PyObject_HEAD sqlite3* db; sqlite3_stmt* st; PyObject* sql; int in_use; + int is_ddl; PyObject* in_weakreflist; /* List of weak references */ } pysqlite_Statement; extern PyTypeObject pysqlite_StatementType; int pysqlite_statement_create(pysqlite_Statement* self, pysqlite_Connection* connection, PyObject* sql); void pysqlite_statement_dealloc(pysqlite_Statement* self);