"not all arguments converted during string formatting" when to_sql

Question

I'm trying the following piece of code, which I found in a 2016 book:

import MySQLdb
import pandas as pd

# database setup omitted for the sake of brevity

nr_customers = 100
colnames = ["movie%i" %i for i in range(1, 33)]
pd.np.random.seed(2015)
generated_customers = pd.np.random.randint(0,2,32 * nr_customers).reshape(nr_customers,32)
data = pd.DataFrame(generated_customers, columns = list(colnames))
data.to_sql('cust',mc,index=True,if_exists='replace',index_label='cust_id')

And it's just giving me the following error:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
~/anaconda3/envs/TestEnv/lib/python3.7/site-packages/MySQLdb/cursors.py in execute(self, query, args)
    242             try:
--> 243                 query = query % args
    244             except TypeError as m:

TypeError: not all arguments converted during string formatting

During handling of the above exception, another exception occurred:

ProgrammingError                          Traceback (most recent call last)
~/anaconda3/envs/TestEnv/lib/python3.7/site-packages/pandas/io/sql.py in execute(self, *args, **kwargs)
   1430             else:
-> 1431                 cur.execute(*args)
   1432             return cur

~/anaconda3/envs/TestEnv/lib/python3.7/site-packages/MySQLdb/cursors.py in execute(self, query, args)
    244             except TypeError as m:
--> 245                 self.errorhandler(self, ProgrammingError, str(m))
    246 

~/anaconda3/envs/TestEnv/lib/python3.7/site-packages/MySQLdb/connections.py in defaulterrorhandler(***failed resolving arguments***)
     51     if errorclass is not None:
---> 52         raise errorclass(errorvalue)
     53     else:

ProgrammingError: not all arguments converted during string formatting

During handling of the above exception, another exception occurred:

DatabaseError                             Traceback (most recent call last)
<ipython-input-24-125bb185f2f4> in <module>
      4 generated_customers = pd.np.random.randint(0,2,32 * nr_customers).reshape(nr_customers,32)
      5 data = pd.DataFrame(generated_customers, columns = list(colnames))
----> 6 data.to_sql('cust',mc,index=True,if_exists='replace',index_label='cust_id')

~/anaconda3/envs/TestEnv/lib/python3.7/site-packages/pandas/core/generic.py in to_sql(self, name, con, schema, if_exists, index, index_label, chunksize, dtype, method)
   2529         sql.to_sql(self, name, con, schema=schema, if_exists=if_exists,
   2530                    index=index, index_label=index_label, chunksize=chunksize,
-> 2531                    dtype=dtype, method=method)
   2532 
   2533     def to_pickle(self, path, compression='infer',

~/anaconda3/envs/TestEnv/lib/python3.7/site-packages/pandas/io/sql.py in to_sql(frame, name, con, schema, if_exists, index, index_label, chunksize, dtype, method)
    458     pandas_sql.to_sql(frame, name, if_exists=if_exists, index=index,
    459                       index_label=index_label, schema=schema,
--> 460                       chunksize=chunksize, dtype=dtype, method=method)
    461 
    462 

~/anaconda3/envs/TestEnv/lib/python3.7/site-packages/pandas/io/sql.py in to_sql(self, frame, name, if_exists, index, index_label, schema, chunksize, dtype, method)
   1544                             if_exists=if_exists, index_label=index_label,
   1545                             dtype=dtype)
-> 1546         table.create()
   1547         table.insert(chunksize, method)
   1548 

~/anaconda3/envs/TestEnv/lib/python3.7/site-packages/pandas/io/sql.py in create(self)
    570 
    571     def create(self):
--> 572         if self.exists():
    573             if self.if_exists == 'fail':
    574                 raise ValueError(

~/anaconda3/envs/TestEnv/lib/python3.7/site-packages/pandas/io/sql.py in exists(self)
    558 
    559     def exists(self):
--> 560         return self.pd_sql.has_table(self.name, self.schema)
    561 
    562     def sql_schema(self):

~/anaconda3/envs/TestEnv/lib/python3.7/site-packages/pandas/io/sql.py in has_table(self, name, schema)
   1556                  "WHERE type='table' AND name={wld};").format(wld=wld)
   1557 
-> 1558         return len(self.execute(query, [name, ]).fetchall()) > 0
   1559 
   1560     def get_table(self, table_name, schema=None):

~/anaconda3/envs/TestEnv/lib/python3.7/site-packages/pandas/io/sql.py in execute(self, *args, **kwargs)
   1443                 "Execution failed on sql '{sql}': {exc}".format(
   1444                     sql=args[0], exc=exc))
-> 1445             raise_with_traceback(ex)
   1446 
   1447     @staticmethod

~/anaconda3/envs/TestEnv/lib/python3.7/site-packages/pandas/compat/__init__.py in raise_with_traceback(exc, traceback)
    418         if traceback == Ellipsis:
    419             _, _, traceback = sys.exc_info()
--> 420         raise exc.with_traceback(traceback)
    421 else:
    422     # this version of raise is a syntax error in Python 3

~/anaconda3/envs/TestEnv/lib/python3.7/site-packages/pandas/io/sql.py in execute(self, *args, **kwargs)
   1429                 cur.execute(*args, **kwargs)
   1430             else:
-> 1431                 cur.execute(*args)
   1432             return cur
   1433         except Exception as exc:

~/anaconda3/envs/TestEnv/lib/python3.7/site-packages/MySQLdb/cursors.py in execute(self, query, args)
    243                 query = query % args
    244             except TypeError as m:
--> 245                 self.errorhandler(self, ProgrammingError, str(m))
    246 
    247         if isinstance(query, unicode):

~/anaconda3/envs/TestEnv/lib/python3.7/site-packages/MySQLdb/connections.py in defaulterrorhandler(***failed resolving arguments***)
     50         raise errorvalue
     51     if errorclass is not None:
---> 52         raise errorclass(errorvalue)
     53     else:
     54         raise Exception(errorvalue)

DatabaseError: Execution failed on sql 'SELECT name FROM sqlite_master WHERE type='table' AND name=?;': not all arguments converted during string formatting

Which I can resume in "DatabaseError: Execution failed on sql 'SELECT name FROM sqlite_master WHERE type='table' AND name=?;': not all arguments converted during string formatting"

I've tried different approaches, like using f"${}" and so on, but the error is the same.

The code isn't completely the same as in the book, since I had to remove the flavor = 'mysql' argument used in to_sql.

I'm using:

• mysql Ver 8.0.15 for osx10.13 on x86_64 (Homebrew)
• Python 3.7.2
• conda 4.6.7
• pandas 0.24.2 py37h0a44026_0
• mysql-connector-c 6.1.11 hccea1a4_0
• mysqlclient 1.3.14 py37h1de35cc_0

Answer 1

Nevermind. Just changed to use sqlalchemy with pymysql and saved a lot of time and LOCs:

from sqlalchemy import create_engine

engine = create_engine('mysql+pymysql://user:password@localhost/database')

...

data.to_sql(table, con = engine)