diff --git a/bin/run b/bin/run
new file mode 100755
index 0000000000..eb7f5ff218
--- /dev/null
+++ b/bin/run
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+source .env
+"$@"
diff --git a/bin/test_multithreading.py b/bin/test_multithreading.py
new file mode 100644
index 0000000000..5eae91623e
--- /dev/null
+++ b/bin/test_multithreading.py
@@ -0,0 +1,63 @@
+"""
+Script to test concurrency (multithreading/multiprocess) issues with the workers. Use with caution.
+"""
+import json
+import atfork
+atfork.monkeypatch_os_fork_functions()
+import atfork.stdlib_fixer
+atfork.stdlib_fixer.fix_logging_module()
+
+import time
+from redash.data import worker
+from redash import models, data_manager, redis_connection
+
+if __name__ == '__main__':
+ models.create_db(True, False)
+
+ print "Creating data source..."
+ data_source = models.DataSource.create(name="Concurrency", type="pg", options="dbname=postgres")
+
+ print "Clear jobs/hashes:"
+ redis_connection.delete("jobs")
+ query_hashes = redis_connection.keys("query_hash_*")
+ if query_hashes:
+ redis_connection.delete(*query_hashes)
+
+ starting_query_results_count = models.QueryResult.select().count()
+ jobs_count = 5000
+ workers_count = 10
+
+ print "Creating jobs..."
+ for i in xrange(jobs_count):
+ query = "SELECT {}".format(i)
+ print "Inserting: {}".format(query)
+ data_manager.add_job(query=query, priority=worker.Job.LOW_PRIORITY,
+ data_source=data_source)
+
+ print "Starting workers..."
+ workers = data_manager.start_workers(workers_count)
+
+ print "Waiting for jobs to be done..."
+ keep_waiting = True
+ while keep_waiting:
+ results_count = models.QueryResult.select().count() - starting_query_results_count
+ print "QueryResults: {}".format(results_count)
+ time.sleep(5)
+ if results_count == jobs_count:
+ print "Yay done..."
+ keep_waiting = False
+
+ data_manager.stop_workers()
+
+ qr_count = 0
+ for qr in models.QueryResult.select():
+ number = int(qr.query.split()[1])
+ data_number = json.loads(qr.data)['rows'][0].values()[0]
+
+ if number != data_number:
+ print "Oops? {} != {} ({})".format(number, data_number, qr.id)
+ qr_count += 1
+
+ print "Verified {} query results.".format(qr_count)
+
+ print "Done."
\ No newline at end of file
diff --git a/manage.py b/manage.py
index f4f2da7e83..3b90cd59a0 100755
--- a/manage.py
+++ b/manage.py
@@ -33,7 +33,7 @@ def runworkers():
logging.info("Cleaning old workers: %s", old_workers)
- data_manager.start_workers(settings.WORKERS_COUNT, settings.CONNECTION_ADAPTER, settings.CONNECTION_STRING)
+ data_manager.start_workers(settings.WORKERS_COUNT)
logging.info("Workers started.")
while True:
@@ -52,6 +52,15 @@ def runworkers():
def make_shell_context():
return dict(app=app, db=db, models=models)
+@manager.command
+def check_settings():
+ from types import ModuleType
+
+ for name in dir(settings):
+ item = getattr(settings, name)
+ if not callable(item) and not name.startswith("__") and not isinstance(item, ModuleType):
+ print "{} = {}".format(name, item)
+
@database_manager.command
def create_tables():
"""Creates the database tables."""
@@ -71,18 +80,23 @@ def drop_tables():
@users_manager.option('name', help="User's full name")
@users_manager.option('--admin', dest='is_admin', action="store_true", default=False, help="set user as admin")
@users_manager.option('--google', dest='google_auth', action="store_true", default=False, help="user uses Google Auth to login")
-def create(email, name, is_admin=False, google_auth=False):
+@users_manager.option('--password', dest='password', default=None, help="Password for users who don't use Google Auth (leave blank for prompt).")
+@users_manager.option('--groups', dest='groups', default=['default'], help="Comma seperated list of groups (leave blank for default).")
+def create(email, name, groups, is_admin=False, google_auth=False, password=None):
print "Creating user (%s, %s)..." % (email, name)
print "Admin: %r" % is_admin
print "Login with Google Auth: %r\n" % google_auth
+ if isinstance(groups, basestring) and len(groups) > 0:
+ groups = groups.split(',')
+ else:
+ groups = ['default']
- permissions = models.User.DEFAULT_PERMISSIONS
if is_admin:
- permissions += ['admin']
+ groups += ['admin']
- user = models.User(email=email, name=name, permissions=permissions)
+ user = models.User(email=email, name=name, groups=groups)
if not google_auth:
- password = prompt_pass("Password")
+ password = password or prompt_pass("Password")
user.hash_password(password)
try:
@@ -101,8 +115,4 @@ def delete(email):
manager.add_command("import", import_manager)
if __name__ == '__main__':
- channel = logging.StreamHandler()
- logging.getLogger().addHandler(channel)
- logging.getLogger().setLevel(settings.LOG_LEVEL)
-
manager.run()
\ No newline at end of file
diff --git a/migrations/add_view_query_permission.py b/migrations/add_view_query_permission.py
new file mode 100644
index 0000000000..7eb9ccc834
--- /dev/null
+++ b/migrations/add_view_query_permission.py
@@ -0,0 +1,13 @@
+import peewee
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+ db.connect_db()
+
+ previous_default_permissions = models.User.DEFAULT_PERMISSIONS[:]
+ previous_default_permissions.remove('view_query')
+ models.User.update(permissions=peewee.fn.array_append(models.User.permissions, 'view_query')).where(peewee.SQL("'view_source' = any(permissions)")).execute()
+
+ db.close_db(None)
diff --git a/migrations/create_data_sources.py b/migrations/create_data_sources.py
new file mode 100644
index 0000000000..45149117b2
--- /dev/null
+++ b/migrations/create_data_sources.py
@@ -0,0 +1,48 @@
+import logging
+import peewee
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+from redash import settings
+
+if __name__ == '__main__':
+ db.connect_db()
+
+ if not models.DataSource.table_exists():
+ print "Creating data_sources table..."
+ models.DataSource.create_table()
+
+ default_data_source = models.DataSource.create(name="Default",
+ type=settings.CONNECTION_ADAPTER,
+ options=settings.CONNECTION_STRING)
+ else:
+ default_data_source = models.DataSource.select().first()
+
+ migrator = Migrator(db.database)
+ models.Query.data_source.null = True
+ models.QueryResult.data_source.null = True
+ try:
+ with db.database.transaction():
+ migrator.add_column(models.Query, models.Query.data_source, "data_source_id")
+ except peewee.ProgrammingError:
+ print "Failed to create data_source_id column -- assuming it already exists"
+
+ try:
+ with db.database.transaction():
+ migrator.add_column(models.QueryResult, models.QueryResult.data_source, "data_source_id")
+ except peewee.ProgrammingError:
+ print "Failed to create data_source_id column -- assuming it already exists"
+
+ print "Updating data source to existing one..."
+ models.Query.update(data_source=default_data_source.id).execute()
+ models.QueryResult.update(data_source=default_data_source.id).execute()
+
+ with db.database.transaction():
+ print "Setting data source to non nullable..."
+ migrator.set_nullable(models.Query, models.Query.data_source, False)
+
+ with db.database.transaction():
+ print "Setting data source to non nullable..."
+ migrator.set_nullable(models.QueryResult, models.QueryResult.data_source, False)
+
+ db.close_db(None)
\ No newline at end of file
diff --git a/migrations/permissions_migration.py b/migrations/permissions_migration.py
new file mode 100644
index 0000000000..5a91621ac0
--- /dev/null
+++ b/migrations/permissions_migration.py
@@ -0,0 +1,24 @@
+from playhouse.migrate import Migrator
+from redash import db
+from redash import models
+
+
+if __name__ == '__main__':
+ db.connect_db()
+ migrator = Migrator(db.database)
+
+ if not models.Group.table_exists():
+ print "Creating groups table..."
+ models.Group.create_table()
+
+ with db.database.transaction():
+ models.Group.insert(name='admin', permissions=['admin'], tables=['*']).execute()
+ models.Group.insert(name='default', permissions=models.Group.DEFAULT_PERMISSIONS, tables=['*']).execute()
+
+ migrator.drop_column(models.User, 'permissions')
+ migrator.add_column(models.User, models.User.groups, 'groups')
+
+ models.User.update(groups=['admin', 'default']).where(models.User.is_admin == True).execute()
+ models.User.update(groups=['default']).where(models.User.is_admin == False).execute()
+
+ db.close_db(None)
diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html
index 4441e0ae95..fae145fc29 100644
--- a/rd_ui/app/index.html
+++ b/rd_ui/app/index.html
@@ -14,6 +14,7 @@
+
@@ -35,7 +36,7 @@
-