Skip to content

Commit 850440b

Browse files
dccotedccote
authored
Merge pull request #151 from DCC-Lab/fix/standard-mysql-url-scheme
Standardize MySQL URL scheme and support inline passwords
2 parents 8fb522e + ecee8b2 commit 850440b

File tree

4 files changed

+89
-91
lines changed

4 files changed

+89
-91
lines changed

‎dcclab/database/database.py‎

Lines changed: 78 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -97,10 +97,11 @@ def __init__(self, databaseURL, usePassword=True, writePermission=False):
9797

9898
self.database = None
9999
self.port = None
100+
self.mysqlPassword = None
100101
self.usePassword = usePassword
101102
self.server = None
102103

103-
self.databaseEngine, self.sshUser, self.sshHost, self.mysqlHost, self.port, self.mysqlUser, self.database = self.parseURL(databaseURL)
104+
self.databaseEngine, self.sshUser, self.sshHost, self.mysqlHost, self.port, self.mysqlUser, self.mysqlPassword, self.database = self.parseURL(databaseURL)
104105

105106
self.connect()
106107

@@ -148,41 +149,44 @@ class color:
148149
print("\n\n")
149150

150151
def parseURL(self, url):
151-
#mysql://sshusername@cafeine2.crulrg.ulaval.ca/mysqlusername:mysqlpassword@questions
152-
match = re.search("mysql://([^@]+?):([^@]+?)/(.*?)@(.+)", url)
153-
if match is not None:
152+
# Standard URL formats:
153+
# mysql://user[:password]@host[:port]/database
154+
# mysql+ssh://sshuser@sshhost/mysqluser[:password]@mysqlhost[:port]/database
155+
# sqlite3://path or file://path
156+
parsed = parse.urlparse(url)
157+
158+
if parsed.scheme == 'mysql':
154159
engine = Engine.mysql
155-
sshUser = None
156-
sshHost = None
157-
mysqlHost = match.group(1)
158-
mysqlPort = match.group(2)
159-
mysqluser = match.group(3)
160-
database = match.group(4)
161-
return (engine, sshUser, sshHost, mysqlHost, mysqlPort, mysqluser, database)
162-
163-
match = re.search("mysql.ssh://(.+?)@([^@]+?):([^@]+?)/(.*?)@(.+)", url)
164-
if match is not None:
160+
mysqlUser = parsed.username
161+
mysqlPassword = parsed.password
162+
mysqlHost = parsed.hostname
163+
mysqlPort = parsed.port or 3306
164+
database = parsed.path.lstrip('/')
165+
if not mysqlUser or not mysqlHost or not database:
166+
raise ValueError("Incomplete mysql URL: {0}. Use mysql://user@host[:port]/database".format(url))
167+
return (engine, None, None, mysqlHost, mysqlPort, mysqlUser, mysqlPassword, database)
168+
169+
if parsed.scheme == 'mysql+ssh':
165170
engine = Engine.mysql
166-
sshUser = match.group(1)
167-
sshHost = match.group(2)
171+
sshUser = parsed.username
172+
sshHost = parsed.hostname
173+
# Path contains: /mysqluser[:password]@mysqlhost[:port]/database
174+
match = re.match(r'^/([^:@]+)(?::([^@]*))?@([^:/]+)(?::(\d+))?/(.+)$', parsed.path)
175+
if match is None:
176+
raise ValueError("Incomplete mysql+ssh URL: {0}. Use mysql+ssh://sshuser@sshhost/mysqluser@mysqlhost/database".format(url))
177+
mysqlUser = match.group(1)
178+
mysqlPassword = match.group(2)
168179
mysqlHost = match.group(3)
169-
mysqlPort = 3306
170-
mysqlUser = match.group(4)
180+
mysqlPort = int(match.group(4)) if match.group(4) else 3306
171181
database = match.group(5)
172-
return (engine, sshUser, sshHost, mysqlHost, mysqlPort, mysqlUser, database)
182+
return (engine, sshUser, sshHost, mysqlHost, mysqlPort, mysqlUser, mysqlPassword, database)
173183

174-
match = re.search(r"(sqlite\d?|file)://(.*?)", url)
175-
if match is not None:
184+
if parsed.scheme in ('sqlite3', 'sqlite', 'file'):
176185
engine = Engine.sqlite3
177-
sshUser = None
178-
sshHost = None
179-
mysqlHost = "127.0.0.1"
180-
mysqlPort = 3306
181-
mysqlUser = None
182-
database = match.group(2)
183-
return (engine, sshUser, sshHost, mysqlHost, mysqlPort, mysqlUser, database)
186+
database = url.split('://', 1)[1]
187+
return (engine, None, None, "127.0.0.1", 3306, None, None, database)
184188

185-
raise ValueError("Unrecognized or incomplete URL: {0}. Use mysql://host/mysqlusername@questions, mysql+ssh://sshusername@sshhost:mysql_host/mysqlusername@questions, or simply sqlite://filename".format(url))
189+
raise ValueError("Unrecognized URL scheme '{0}' in: {1}. Use mysql://user@host[:port]/database, mysql+ssh://sshuser@sshhost/mysqluser@mysqlhost/database, or sqlite3://path".format(parsed.scheme, url))
186190

187191
def __enter__(self):
188192
return self
@@ -209,9 +213,12 @@ def connect(self):
209213
self.cursor = self.connection.cursor()
210214
else:
211215
import mysql.connector as mysql
212-
import keyring
213216

214-
if self.usePassword is True:
217+
if self.mysqlPassword is not None:
218+
pwd = self.mysqlPassword
219+
elif self.usePassword is True:
220+
import keyring
221+
215222
if self.sshHost is not None:
216223
serviceName = "mysql-{0}-ssh-{1}".format(self.mysqlHost, self.sshHost)
217224
else:
@@ -220,7 +227,8 @@ def connect(self):
220227
pwd = keyring.get_password(serviceName, self.mysqlUser)
221228
if pwd is None:
222229
raise Exception(""" Set the password in the system password manager on the command line with:
223-
{2} -m keyring set {0} {1}""".format(serviceName, self.mysqlUser, sys.executable))
230+
{2} -m keyring set {0} {1}
231+
Or provide the password in the URL: mysql://user:password@host/database""".format(serviceName, self.mysqlUser, sys.executable))
224232
else:
225233
pwd = None
226234

@@ -230,8 +238,6 @@ def connect(self):
230238
self.server = Cafeine(username=self.sshUser)
231239
self.port = self.server.startMySQLTunnel(ssh_host=self.sshHost, remote_bind_address=self.mysqlHost)
232240
actualMysqlHost = "127.0.0.1"
233-
else:
234-
self.port = 3306
235241

236242
self.connection = mysql.connect(host=actualMysqlHost,
237243
port=self.port,
@@ -478,62 +484,50 @@ def __init__(self, databaseURL, usePassword=True, writePermission=False):
478484

479485
self.database = None
480486
self.port = None
487+
self.mysqlPassword = None
481488
self.usePassword = usePassword
482489
self.server = None
483490

484-
self.databaseEngine, self.sshUser, self.sshHost, self.mysqlHost, self.port, self.mysqlUser, self.database = self.parseURL(databaseURL)
491+
self.databaseEngine, self.sshUser, self.sshHost, self.mysqlHost, self.port, self.mysqlUser, self.mysqlPassword, self.database = self.parseURL(databaseURL)
485492

486493
self.connect()
487494

488495
def showDatabaseInfo(self):
489496
pass
490497

491498
def parseURL(self, url):
492-
match = re.search("mysql://([^@]+?):([^@]+?)/(.*?)@(.+)", url)
493-
if match is not None:
499+
# Standard URL formats:
500+
# mysql://user[:password]@host[:port]/database
501+
# mysql+ssh://sshuser@sshhost/mysqluser[:password]@mysqlhost[:port]/database
502+
parsed = parse.urlparse(url)
503+
504+
if parsed.scheme == 'mysql':
494505
engine = Engine.mysql
495-
sshUser = None
496-
sshHost = None
497-
mysqlHost = match.group(1)
498-
mysqlPort = match.group(2)
499-
mysqluser = match.group(3)
500-
database = match.group(4)
501-
return (engine, sshUser, sshHost, mysqlHost, mysqlPort, mysqluser, database)
502-
503-
match = re.search("mysql.ssh://(.+?)@([^@]+?):([^@]+?)/(.*?)@(.+)", url)
504-
if match is not None:
506+
mysqlUser = parsed.username
507+
mysqlPassword = parsed.password
508+
mysqlHost = parsed.hostname
509+
mysqlPort = parsed.port or 3306
510+
database = parsed.path.lstrip('/')
511+
if not mysqlUser or not mysqlHost or not database:
512+
raise ValueError("Incomplete mysql URL: {0}. Use mysql://user@host[:port]/database".format(url))
513+
return (engine, None, None, mysqlHost, mysqlPort, mysqlUser, mysqlPassword, database)
514+
515+
if parsed.scheme == 'mysql+ssh':
505516
engine = Engine.mysql
506-
507-
sshUser = match.group(1)
508-
sshHost = match.group(2)
517+
sshUser = parsed.username
518+
sshHost = parsed.hostname
519+
# Path contains: /mysqluser[:password]@mysqlhost[:port]/database
520+
match = re.match(r'^/([^:@]+)(?::([^@]*))?@([^:/]+)(?::(\d+))?/(.+)$', parsed.path)
521+
if match is None:
522+
raise ValueError("Incomplete mysql+ssh URL: {0}. Use mysql+ssh://sshuser@sshhost/mysqluser@mysqlhost/database".format(url))
523+
mysqlUser = match.group(1)
524+
mysqlPassword = match.group(2)
509525
mysqlHost = match.group(3)
510-
mysqlPort = 3306
511-
mysqlUser = match.group(4)
526+
mysqlPort = int(match.group(4)) if match.group(4) else 3306
512527
database = match.group(5)
513-
return (engine, sshUser, sshHost, mysqlHost, mysqlPort, mysqlUser, database)
514-
515-
# #mysql://sshusername@cafeine2.crulrg.ulaval.ca/mysqlusername:mysqlpassword@questions
516-
# match = re.search("mysql://([^@]+?)/(.*?)@(.+)", url)
517-
# if match is not None:
518-
# engine = Engine.mysql
519-
# sshUser = None
520-
# sshHost = None
521-
# mysqlHost = match.group(1)
522-
# mysqluser = match.group(2)
523-
# database = match.group(3)
524-
# return (engine, sshUser, sshHost, mysqlHost, mysqluser, database)
525-
526-
# match = re.search("mysql.ssh://(.+?)@([^@]+?):([^@]+?)/(.*?)@(.+)", url)
527-
# if match is not None:
528-
# engine = Engine.mysql
529-
# sshUser = match.group(1)
530-
# sshHost = match.group(2)
531-
# mysqlHost = match.group(3)
532-
# mysqlUser = match.group(4)
533-
# database = match.group(5)
534-
# return (engine, sshUser, sshHost, mysqlHost, mysqlUser, database)
535-
536-
raise ValueError("Unrecognized or incomplete URL: {0}. Use mysql://host/mysqlusername@questions, mysql+ssh://sshusername@sshhost:mysql_host/mysqlusername@questions, or simply sqlite://filename".format(url))
528+
return (engine, sshUser, sshHost, mysqlHost, mysqlPort, mysqlUser, mysqlPassword, database)
529+
530+
raise ValueError("Unrecognized URL scheme '{0}' in: {1}. Use mysql://user@host[:port]/database or mysql+ssh://sshuser@sshhost/mysqluser@mysqlhost/database".format(parsed.scheme, url))
537531

538532
def __enter__(self):
539533
return self
@@ -546,9 +540,12 @@ def connect(self):
546540
try:
547541
if not self.isConnected:
548542
import mysql.connector as mysql
549-
import keyring
550543

551-
if self.usePassword is True:
544+
if self.mysqlPassword is not None:
545+
pwd = self.mysqlPassword
546+
elif self.usePassword is True:
547+
import keyring
548+
552549
if self.sshHost is not None:
553550
serviceName = "mysql-{0}-ssh-{1}".format(self.mysqlHost, self.sshHost)
554551
else:
@@ -559,7 +556,8 @@ def connect(self):
559556
pwd = keyring.get_password("mysql-{0}".format(self.mysqlHost), self.mysqlUser)
560557
if pwd is None:
561558
raise Exception(""" Set the password in the system password manager on the command line with:
562-
{2} -m keyring set {0} {1}""".format(serviceName, self.mysqlUser, sys.executable))
559+
{2} -m keyring set {0} {1}
560+
Or provide the password in the URL: mysql://user:password@host/database""".format(serviceName, self.mysqlUser, sys.executable))
563561
else:
564562
pwd = None
565563

@@ -777,7 +775,7 @@ def insert(self, table: str, entry: dict):
777775

778776
if __name__ == "__main__":
779777
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')))
780-
db = Database("mysql+ssh://dcclab@cafeine3.crulrg.ulaval.ca/dcclab@labdata")
778+
db = Database("mysql+ssh://dcclab@cafeine3.crulrg.ulaval.ca/dcclab@cafeine3.crulrg.ulaval.ca/labdata")
781779
db.execute('select * from files')
782780
print(db.fetchAll())
783781
# db = Database("/Users/dccote/GitHub/PyVino/raman.db")

‎dcclab/database/labdatadb.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,22 +32,22 @@ class LabdataDB(MySQLDatabase):
3232
secure shell through cafeine2, and then via mysql also with dcclab and the same password.
3333
The database is called labdata, and the default value of the URL
3434
to access it is set to:
35-
mysql+ssh://dcclab@cafeine2.crulrg.ulaval.ca:cafeine3.crulrg.ulaval.ca/dcclab@labdata
35+
mysql+ssh://dcclab@cafeine2.crulrg.ulaval.ca/dcclab@cafeine3.crulrg.ulaval.ca/labdata
3636
which can be interpreted as:
37-
mysql://ssh_username@ssh_host:mysql_host/mysql_user@mysql_database
37+
mysql+ssh://ssh_user@ssh_host/mysql_user@mysql_host/database
3838
3939
You can provide your own link if you have a local version on your computer, such as:
40-
db = LabdataDB("mysql://127.0.0.1:3308/dcclab@labdata")
40+
db = LabdataDB("mysql://dcclab@127.0.0.1:3308/labdata")
4141
"""
4242
def __init__(self, databaseURL=None):
4343
"""
4444
The Database is initialized to:
45-
mysql+ssh://dcclab@cafeine2.crulrg.ulaval.ca:cafeine3.crulrg.ulaval.ca/dcclab@labdata
45+
mysql+ssh://dcclab@cafeine2.crulrg.ulaval.ca/dcclab@cafeine3.crulrg.ulaval.ca/labdata
4646
4747
which creates an SSH tunnel through cafeine2 to reach the MySQL server on cafeine3.
4848
"""
4949
if databaseURL is None:
50-
databaseURL = "mysql+ssh://dcclab@cafeine2.crulrg.ulaval.ca:cafeine3.crulrg.ulaval.ca/dcclab@labdata"
50+
databaseURL = "mysql+ssh://dcclab@cafeine2.crulrg.ulaval.ca/dcclab@cafeine3.crulrg.ulaval.ca/labdata"
5151

5252
self.constraints = []
5353
super().__init__(databaseURL)

‎dcclab/tests/testDatabase.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,7 @@ class TestMySQLDatabase(env.DCCLabTestCase):
246246
def setUp(self):
247247
super().setUp()
248248

249-
self.db = MySQLDatabase("mysql+ssh://dcclab@cafeine2.crulrg.ulaval.ca:cafeine3.crulrg.ulaval.ca/dcclab@labdata")
249+
self.db = MySQLDatabase("mysql+ssh://dcclab@cafeine2.crulrg.ulaval.ca/dcclab@cafeine3.crulrg.ulaval.ca/labdata")
250250

251251
self.assertIsNotNone(self.db)
252252
self.db.connect()

‎dcclab/tests/testLabdata.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,17 +28,17 @@ def testConnectDBBadURL(self):
2828

2929
def testConnectDBBadHost(self):
3030
with self.assertRaises(Exception):
31-
db = LabdataDB("mysql://somehost")
31+
db = LabdataDB("mysql://user@somehost/db")
3232
db.disconnect()
3333

3434
# def testConnectDBGoodHost(self):
35-
# db = LabdataDB("mysql://127.0.0.1/root@labdata")
35+
# db = LabdataDB("mysql://root@127.0.0.1/labdata")
3636
# db.disconnect()
3737

3838
def testLocalConnectOnCafeine2(self):
3939
if self.isAtCERVO():
4040
with self.assertRaises(Exception): # access denied, only localhost as of May 17th
41-
db = LabdataDB("mysql://cafeine3.crulrg.ulaval.ca/dcclab@labdata")
41+
db = LabdataDB("mysql://dcclab@cafeine3.crulrg.ulaval.ca/labdata")
4242
else:
4343
self.skipTest("Not at CERVO: skipping local connections")
4444

@@ -62,7 +62,7 @@ def testIsAtCervo(self):
6262

6363
def testLocalConnectOnCafeine3(self):
6464
if self.isAtCERVO():
65-
db = LabdataDB("mysql://cafeine3.crulrg.ulaval.ca/dcclab@labdata")
65+
db = LabdataDB("mysql://dcclab@cafeine3.crulrg.ulaval.ca/labdata")
6666
else:
6767
self.skipTest("Not at CERVO: skipping local connections")
6868

@@ -283,7 +283,7 @@ def testGetSpectralDataFrom(self):
283283

284284
# class TestMySQLDatabase(env.DCCLabTestCase):
285285
# def testLocalMySQLDatabase(self):
286-
# db = Database("mysql://127.0.0.1/root@raman")
286+
# db = Database("mysql://root@127.0.0.1/raman")
287287
# db.execute("select * from spectra where datatype = 'raw'")
288288
#
289289
# rows = []

0 commit comments

Comments
 (0)