Not too bad limited Beta

Author: durch

Cargo.toml

@@ -20,5 +20,7 @@ protobuf = "1.0.24"
 rustc-serialize = "0.3.22"
 goauth = "0.2.12"
 
+lazy_static = "0.2.4"
+
 [build-dependencies]
 gcc = "*"

Makefile

@@ -1,5 +1,5 @@
 EXTENSION    = bigtable
-EXTVERSION   = 0.0.1
+EXTVERSION   = 0.1.0
 
 DATA         = sql/$(EXTENSION)--$(EXTVERSION).sql
 PG_CONFIG    = pg_config

README.md

@@ -2,87 +2,75 @@
 
 # Google Bigtable Rust PostgreSQL FDW
 
-## IGNORE DOCS BELOW, PROPER ONES COMING SOON :)
+[Rust](https://www.rust-lang.org/en-US/) [PostgreSQL](https://www.postgresql.org/) foreign data wrapper for interfacing with [Google Cloud Bigtable](https://cloud.google.com/bigtable/), as well as other API compatible databases ([HBase](https://hbase.apache.org/) should work with some effort).
 
-[Rust](https://www.rust-lang.org/en-US/) [PostgreSQL](https://www.postgresql.org/) extension for interfacing with [Google Cloud Bigtable](https://cloud.google.com/bigtable/), as well as other API compatible databases ([HBase](https://hbase.apache.org/) should work with some effort).
-
-While logic is contained in `Rust`, it leverages `PostgreSQL ` `C` macros for passing parameters around as well returning values, and for ease of use it is all wrapped in `PL/pgSQL`.
-
-At the moment **reading** and **writing** form/to `Bigtable` is supported with *deleting* and *updating* on the roadmap, as well as a few other more Bigtable oriented features, and encrypted credential storage. At present it is more of an exercise in writing a `PostgreSQL` extension in `Rust` than anything else.
+While logic is contained in `Rust`, it leverages `PostgreSQL ` `C` FDW callbacks.
 
+### Roadmap
+[x] `select`
+[x] `select limit`
+[ ] `select offset`
+[ ] `select where`
+[x] `insert`
+[ ] `update`
+[ ] `delete`
 
 ## Installation
-+ `PostgreSQL 9.3+`
++ `PostgreSQL 9.6+`
 + `Stable Rust 1.15+`, get it using [rustup](https://www.rustup.rs/).
 
 ```bash
-git clone https://github.com/durch/google-bigtable-postgres-extension.git
-cd google-bigtable-postgres-extension
+git clone https://github.com/durch/google-bigtable-postgres-fdw.git
+cd google-bigtable-postgres-fdw
 make install
 psql -U postgres
 ```
 
-Once inside the DB
+### Initial DB setup
 
-```sql
+```plpgsql
 CREATE EXTENSION bigtable;
+CREATE SERVER test FOREIGN DATA WRAPPER bigtable OPTIONS (instance '`instance_id`', project '`project_id`');
+CREATE FOREIGN TABLE test(bt json) SERVER test OPTIONS (name '`table_name`');
+CREATE USER MAPPING FOR postgres SERVER TEST OPTIONS (credentials_path '`path_to_service_account_json_credentials`');
 ```
 
-The command above will also output a message about inputing your credentials, which you can get from Google Cloud Console, the intention is to work using service accounts, you'll need proper have [scopes](https://cloud.google.com/bigtable/docs/creating-compute-instance) in order to be able to use the extension. Once you have the `json` credential file, you can feed it in:
-
-```sql
-SELECT bt_set_credentials('<absolute_path_to_gcloud_json_credentials>');
-```
-
-The contents of the file are read, `base64` encoded and stored in `bt_auth_config` table, this is completely insecure and you should take care not allow bad guys access to this key, especially if it has admin scopes.
-
-You can delete the file from the system after reading it in, it will be dumped and restored along with other `DB` tables.
+### Usage
 
-## Usage
+You can use [gen.py]() to generate some test data. Modify `gen.py` to adjust for the number of generated records, also modify the`column` key in the generated output as this needs be a `column familly` that **exists** in your Bigtable, running `python gen.py` outputs `test.sql`, which can be fed into PG. `WHERE` is evaluted on the PG side so be sure to grab what you need from BT.
 
-### Reading
+```
+psql -U postgres < test.sql
+```
 
-```sql
-# SIGNATURE
-bt_read_rows(instance_name TEXT, table_name TEXT, limit INT) RETURNS JSON
+#### SELECT
 
-# EXAMPLE
-# Reading 10 rows from test_table in test-instance
-SELECT bt_read_rows('test-instance', 'test-table', 10);
+One Bigtable row per PG rowis returned, limit is done on the BT side, rows are returned as `json` and can be further manipulated using Postgres `json` [functions and operators](`https://www.postgresql.org/docs/9.6/static/functions-json.html`).
 
-# Output will be valid json, it will fail if the table is empty
 ```
+SELECT * FROM test;
+SELECT * FROM test LIMIT 100;
 
-### Writing
-
-#### Writing one row at a time
-
-```sql
-# SIGNATURES
-bt_write_one(column_family TEXT, column_qulifier TEXT, rows TEXT, instance_name TEXT, table_name TEXT) RETURNS TEXT
+SELECT bt->'familyName', bt->'qualifier' FROM test WHERE bt->>'rowKey' ~* '.*regex.*';
+SELECT bt->'familyName', bt->'qualifier' FROM test WHERE bt->>'rowKey' = 'exact';
+```
 
-bt_write_one(column_family TEXT, column_qulifier TEXT, rows JSON, instance_name TEXT, table_name TEXT) RETURNS TEXT
+#### INSERT
 
-# EXAMPLES
-SELECT bt_write_one('cf1', 't', 'Sample text row', 'test-instance','test-table');
+`INSERT` format is a bit weird ATM:
 
-SELECT bt_write_one('cf1', 't', '"Sample json text row"', 'test-instance','test-table');
+```json
 
-SELECT bt_write_one('cf1', 't', '{"json_object_row": true}', 'test-instance','test-table');
+{
+    "row_key": string,
+    "column": string,
+    "column_qualifier": string,
+    "data": [
+        json
+    ]
+}
 
-SELECT bt_write_one('cf1', 't', '["json", "array", "row"]', 'test-instance','test-table');
 ```
 
-#### Writing many rows at once
-
-In the case of `bt_write_many` `json` *arrays* are unpacked into rows, all other `json` types will be written as one row, as if using `bt_write_one`. 
+Currently `row_key` is treated as a prefix and concated with a loop counter, while this covers a few use cases it is not really ideal for Bigtable. This will likely be extended to allow passing of a `row_key` array. As you are passing in one `json` object which gets expanded, `INSERT` counter always shows one row inserted, truth can be found in PG logs.
 
-This should enable straightforward import of data to Bigtable as PostgreSQL has some very nice [JSON functions](https://www.postgresql.org/docs/9.6/static/functions-json.html) for formatting and converting data.
-
-```sql
-# SIGNATURE
-bt_write_many(column_family TEXT, column_qulifier TEXT, rows JSON, instance_name IN TEXT, table_name) RETURNS TEXT
-
-# EXAMPLE
-SELECT bt_write_many('cf1', 't', '["this", "will", "make", 5, "rows"]', 'test-instance', 'test-table');
-```

bigtable.control

@@ -1,5 +1,5 @@
 # base36 extension in rust
 comment = 'Rust extension for interfaceing with Google Bigtable from PostgreSQL'
-default_version = '0.0.1'
+default_version = '0.1.0'
 module_pathname = '$libdir/bigtable_pg_ext'
 relocatable = true

gen.py

@@ -2,13 +2,21 @@
 import random
 import string
 
+N_RECORDS = 1000 # number of records generated
+OUT_FILE = "test.sql"
+
+# Be sure to modify the `column key below`
+
 def id_gen(N=64):
     return ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(N))
 
 def coord_gen(rng=90):
     return random.randint(rng*-1, rng)
 
-l = [{"id": id_gen(), "lat": coord_gen(90), "lng": coord_gen(180)} for _ in xrange(100000)]
-dt = json.dumps({"row_key": id_gen(), "column": id_gen(), "column_qualifier": id_gen(), "data": l})
-with open("test.sql", "wb") as fp:
-    fp.write("insert into test values('{}');".format(dt))
+if __name__ = "__main__":
+
+    l = [{"id": id_gen(), "lat": coord_gen(90), "lng": coord_gen(180)} for _ in xrange(N_RECORDS)]
+
+    dt = json.dumps({"row_key": id_gen(), "column": 'cf1', "column_qualifier": 'test', "data": l})
+    with open(OUT_FILE, "wb") as fp:
+        fp.write("insert into test values('{}');".format(dt))

sql/bigtable.sql

@@ -1,7 +1,3 @@
--- drop extension bigtable cascade; create extension bigtable; create server test foreign data wrapper bt_fdw options (instance 'testinst', project 'drazens-bigtable-testing'); create foreign table test(bt json) server test options (name 'test'); create user mapping for postgres server test options (credentials_path '/tmp/code/drazens-bigtable-testing-9f32bd2aa193.json');
---
---
-
 -- -- complain if script is sourced in psql, rather than via CREATE EXTENSION
 \echo Use "CREATE EXTENSION bigtable" to load this file. \quit
 
@@ -15,6 +11,6 @@ RETURNS void
 AS '$libdir/bigtable'
 LANGUAGE C STRICT;
 
-CREATE FOREIGN DATA WRAPPER bt_fdw
+CREATE FOREIGN DATA WRAPPER bigtable
   HANDLER bt_fdw_handler
   VALIDATOR bt_fdw_validator;

src/bt_fdw.c

@@ -49,6 +49,7 @@ btGetForeignRelSize(PlannerInfo *root,
                     RelOptInfo *baserel,
                     Oid foreigntableid) {
     elog(LOG, "entering function %s", __func__);
+
     baserel->rows = 500;
 }
 
@@ -87,6 +88,8 @@ btGetForeignPlan(PlannerInfo *root,
 
     Index scan_relid = baserel->relid;
 
+    get_limit(root);
+
     scan_clauses = extract_actual_clauses(scan_clauses, false);
 
     return make_foreignscan(tlist,

src/bt_fdw.h

@@ -97,6 +97,9 @@ bt_fdw_iterate_foreign_scan(bt_fdw_state_t *, ForeignScanState *);
 extern void *
 bt_fdw_exec_foreign_insert(bt_fdw_state_t *, TupleTableSlot *, char *);
 
+extern void *
+get_limit(PlannerInfo *);
+
 /*
  * FDW functions declarations
  */