![]() ![]() Server_1 | return super(BaseResource, self).dispatch_request(*args, **kwargs) Server_1 | File "/app/redash/handlers/base.py", line 33, in dispatch_request Server_1 | return self.dispatch_request(*args, **kwargs) Server_1 | File "/usr/local/lib/python3.7/site-packages/flask/views.py", line 89, in view Server_1 | File "/usr/local/lib/python3.7/site-packages/flask_login/utils.py", line 261, in decorated_view Server_1 | resp = resource(*args, **kwargs) Server_1 | File "/usr/local/lib/python3.7/site-packages/flask_restful/_init_.py", line 458, in wrapper Server_1 | return self.view_functions(**req.view_args) Server_1 | File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1935, in dispatch_request Server_1 | File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1949, in full_dispatch_request Server_1 | File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1820, in handle_user_exception Server_1 | rv = self.handle_user_exception(e) Server_1 | File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1951, in full_dispatch_request Server_1 | response = self.full_dispatch_request() Server_1 | File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2446, in wsgi_app Server_1 | raise value.with_traceback(tb) Server_1 | File "/usr/local/lib/python3.7/site-packages/flask/_compat.py", line 38, in reraise Server_1 | reraise(exc_type, exc_value, tb) Server_1 | File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1866, in handle_exception Server_1 | File "/usr/local/lib/python3.7/site-packages/flask_restful/_init_.py", line 269, in error_router Server_1 | response = self.handle_exception(e) Server_1 | File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2449, in wsgi_app Server_1 | return self.app(environ, start_response) Server_1 | File "/usr/local/lib/python3.7/site-packages/werkzeug/middleware/proxy_fix.py", line 232, in _call_ Server_1 | return gi_app(environ, start_response) Server_1 | File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2463, in _call_ The error message that I get is server_1 | Traceback (most recent call last): I tested with different clients, such as DataGrip or Postico, and it does work fine on that. LOCATION '/datasets/iris-dataset.I tried to connect BigQuery but it does not allow me to connect using Service Account. CREATE TABLE IF NOT EXISTS iris_dataset( `sl` double, `sw` double, `pl` double, `pw` double ) To do so you need to execute Spark's saveAsTable() function.Īnother option is to directly create the tables from external files (such as parquet or CSV) from the external SQL tool.įor example to create a new table execute a CREATE TABLE. In both situations they need to be "registered" in the metastore. Tables are created either through an import process using a Reusable Code Block, or created via a Jupyter notebook. Depending on your use case you might need to add more RAM to support more complex joins. SparkSQL scales horizontally so if the performance is not satisfactory add more workers from SparkSQL's Configuration Tab. Execute some queries on the connection:.If not, check your firewall settings at step 2. In DataGrip, click the "+" sign and add a Data Source by selecting the newly added Hive 1.2.1. In the same dialogue, at the Firewall tab make sure your IP is white listed from your current location. Click on the SparkSQL's Edit button and copy the JDBC URL.On the Options select the Apache Spark option in both Dialect and Icon dropdowns. Click on the "+" sign from "Driver files" and add both jarsĬhange the class to ".HiveDriver".Click on the "+" sign and select "Driver":.For the purpose of this demonstration we're going to use Jetbrains's excelent DataGrip. The JDBC connectors should work with all JDBC compatible clients. Configure your BI tool to use the JDBC drivers mkdir ~/jdbc-drivers #you can put these anywhere cd ~/jdbc-driversģ. It also has a Hadoop-core dependency that does not come with it. SparkSQL is compatible with Apache Hive's JDBC connector version 1.x. Deploy the Spark SQL Applicationįrom the Lentiq's left-hand application panel click on the SparkSQL icon.Ĭlick Create Spark SQL. The query engine is SparkSQL which uses Spark's in-memory mechanisms and query planner to execute SQL queries on data. The data is stored in parquet format in the object storage, the schema is stored a metastore database that is linked to Lentiq's meta data management system. Lentiq is compatible with most JDBC/ODBC compatible tools and uses Apache Spark's query engine. Querying data using SQL is a basic but fundamental use of any data lake. ![]()
0 Comments
Leave a Reply. |
AuthorWrite something about yourself. No need to be fancy, just an overview. ArchivesCategories |