Merge pull request #226 from weiznich/sqlite_infer_schema

Implement infer_schema for sqlite

.travis.yml

@@ -22,14 +22,14 @@ script:
   fi &&
   (cd diesel_cli && travis-cargo test -- --no-default-features --features "$BACKEND") &&
   if [[ "$TRAVIS_RUST_VERSION" == nightly* ]]; then
-    (cd diesel_codegen && travis-cargo test -- --no-default-features --features nightly)
+    (cd diesel_codegen && travis-cargo test -- --no-default-features --features "nightly $BACKEND")
   else
-    (cd diesel_codegen && travis-cargo test)
+    (cd diesel_codegen && travis-cargo test -- --no-default-features --features "with-syntex $BACKEND")
   fi &&
   if [[ "$TRAVIS_RUST_VERSION" == nightly* ]]; then
     (cd diesel_tests && travis-cargo test -- --no-default-features --features "unstable $BACKEND")
   else
-    (cd diesel_tests && travis-cargo test -- --features $BACKEND)
+    (cd diesel_tests && travis-cargo test -- --no-default-features --features "syntex dotenv_codegen $BACKEND")
   fi &&
   if [[ "$TRAVIS_RUST_VERSION" == nightly* ]]; then
     (cd diesel_compile_tests && travis-cargo test)

diesel/src/sqlite/connection/mod.rs

@@ -132,6 +132,16 @@ impl SqliteConnection {
         }
         query.map(|_| ())
     }
+
+    #[doc(hidden)]
+    pub fn execute_pragma<ST, U>(&self, source: &str) -> QueryResult<Vec<U>> where
+        U: Queryable<ST, Sqlite>,
+        <SqliteConnection as Connection>::Backend: HasSqlType<ST>,
+    {
+        Statement::prepare(&self.raw_connection, source)
+            .map(StatementIterator::new)
+            .and_then(Iterator::collect)
+    }
 }
 
 fn error_message(err_code: libc::c_int) -> &'static str {

diesel_codegen/src/lib.in.rs

@@ -3,10 +3,6 @@ mod attr;
 mod insertable;
 mod model;
 mod queryable;
-#[cfg(feature = "postgres")]
-mod schema_inference;
-#[cfg(not(feature = "postgres"))]
-#[path="dummy_schema_inference.rs"]
 mod schema_inference;
 
 mod update;

diesel_codegen/src/schema_inference/data_structures.rs

@@ -1,54 +1,61 @@
 use diesel::*;
+#[cfg(feature = "postgres")]
 use diesel::pg::Pg;
+#[cfg(feature = "postgres")]
+use diesel::pg::PgConnection;
+#[cfg(feature = "sqlite")]
+use diesel::sqlite::Sqlite;
+#[cfg(feature = "sqlite")]
+use diesel::sqlite::SqliteConnection;
 use diesel::types::{HasSqlType, FromSqlRow};
 
-table! {
-    pg_attribute (attrelid) {
-        attrelid -> Oid,
-        attname -> VarChar,
-        atttypid -> Oid,
-        attnotnull -> Bool,
-        attnum -> SmallInt,
-        attisdropped -> Bool,
-    }
-}
-
-table! {
-    pg_type (oid) {
-        oid -> Oid,
-        typname -> VarChar,
-    }
-}
-
-joinable!(pg_attribute -> pg_type (atttypid));
-select_column_workaround!(pg_attribute -> pg_type (attrelid, attname, atttypid, attnotnull, attnum, attisdropped));
-select_column_workaround!(pg_type -> pg_attribute (oid, typname));
-
-table! {
-    pg_class (oid) {
-        oid -> Oid,
-        relname -> VarChar,
-    }
-}
-
 #[derive(Debug, Clone)]
-pub struct PgAttr {
+pub struct ColumnInformation {
     pub column_name: String,
     pub type_name: String,
     pub nullable: bool,
 }
 
-impl<ST> Queryable<ST, Pg> for PgAttr where
+#[cfg(feature = "postgres")]
+impl<ST> Queryable<ST, Pg> for ColumnInformation where
     Pg: HasSqlType<ST>,
     (String, String, bool): FromSqlRow<ST, Pg>,
 {
     type Row = (String, String, bool);
 
     fn build(row: Self::Row) -> Self {
-        PgAttr {
+        ColumnInformation {
             column_name: row.0,
             type_name: row.1,
             nullable: !row.2,
         }
     }
 }
+
+#[cfg(feature = "sqlite")]
+impl<ST> Queryable<ST, Sqlite> for ColumnInformation where
+    Sqlite: HasSqlType<ST>,
+    (i32, String, String, bool, Option<String>, i32): FromSqlRow<ST, Sqlite>,
+{
+    type Row = (i32, String, String, bool, Option<String>, i32);
+
+    fn build(row: Self::Row) -> Self {
+        ColumnInformation {
+            column_name: row.1,
+            type_name: row.2,
+            nullable: !row.3,
+        }
+    }
+}
+
+#[cfg(all(feature = "sqlite", not(feature = "postgres")))]
+pub type InferConnection = SqliteConnection;
+
+#[cfg(all(feature = "postgres", not(feature = "sqlite")))]
+pub type InferConnection = PgConnection;
+
+#[cfg(all(feature = "sqlite", feature = "postgres"))]
+pub enum InferConnection{
+    Sqlite(SqliteConnection),
+    Pg(PgConnection),
+}

diesel_codegen/src/schema_inference/mod.rs

@@ -1,7 +1,10 @@
 mod data_structures;
+#[cfg(feature = "postgres")]
+mod pg;
+#[cfg(feature = "sqlite")]
+mod sqlite;
 
-use diesel::*;
+use diesel::{QueryResult, Connection};
-use diesel::pg::PgConnection;
 use syntax::ast;
 use syntax::codemap::Span;
 use syntax::ext::base::*;
@@ -73,25 +76,14 @@ pub fn infer_schema_body<T: Iterator<Item=P<ast::Expr>>>(
     Ok(MacEager::items(SmallVector::many(try!(impls))))
 }
 
-fn establish_connection(
-    cx: &mut ExtCtxt,
-    sp: Span,
-    database_url: &str,
-) -> Result<PgConnection, Box<MacResult>> {
-    PgConnection::establish(database_url).map_err(|_| {
-        cx.span_err(sp, "failed to establish a database connection");
-        DummyResult::any(sp)
-    })
-}
-
 fn table_macro_call(
     cx: &mut ExtCtxt,
     sp: Span,
-    connection: &PgConnection,
+    connection: &InferConnection,
     table_name: &str,
 ) -> Result<P<ast::Item>, Box<MacResult>> {
     match get_table_data(connection, table_name) {
-        Err(NotFound) => {
+        Err(::diesel::result::Error::NotFound) => {
             cx.span_err(sp, &format!("no table exists named {}", table_name));
             Err(DummyResult::any(sp))
         }
@@ -100,7 +92,7 @@ fn table_macro_call(
             Err(DummyResult::any(sp))
         }
         Ok(data) => {
-            let tokens = data.iter().map(|a| column_def_tokens(cx, a))
+            let tokens = data.iter().map(|a| column_def_tokens(cx, a, &connection))
                 .collect::<Vec<_>>();
             let table_name = str_to_ident(table_name);
             let item = quote_item!(cx, table! {
@@ -124,60 +116,137 @@ fn next_str_lit<T: Iterator<Item=P<ast::Expr>>>(
     }
 }
 
-fn get_table_data(conn: &PgConnection, table_name: &str) -> QueryResult<Vec<PgAttr>> {
+fn column_def_tokens(cx: &mut ExtCtxt, attr: &ColumnInformation, conn: &InferConnection)
-    use self::data_structures::pg_attribute::dsl::*;
+    -> Vec<ast::TokenTree>
-    use self::data_structures::pg_type::dsl::{pg_type, typname};
+{
-    let t_oid = try!(table_oid(conn, table_name));
-
-    pg_attribute.inner_join(pg_type)
-        .select((attname, typname, attnotnull))
-        .filter(attrelid.eq(t_oid))
-        .filter(attnum.gt(0).and(attisdropped.ne(true)))
-        .order(attnum)
-        .load(conn)
-}
-
-fn table_oid(conn: &PgConnection, table_name: &str) -> QueryResult<u32> {
-    use self::data_structures::pg_class::dsl::*;
-    pg_class.select(oid).filter(relname.eq(table_name)).first(conn)
-}
-
-fn column_def_tokens(cx: &mut ExtCtxt, attr: &PgAttr) -> Vec<ast::TokenTree> {
     let column_name = str_to_ident(&attr.column_name);
-    let tpe = determine_column_type(cx, attr);
+    let tpe = determine_column_type(cx, attr, conn);
     quote_tokens!(cx, $column_name -> $tpe,)
 }
 
-fn determine_column_type(cx: &mut ExtCtxt, attr: &PgAttr) -> P<ast::Ty> {
+fn establish_real_connection<Conn>(
-    let tpe;
+    cx: &mut ExtCtxt,
-    if attr.type_name.starts_with("_") {
+    sp: Span,
-        let subtype = str_to_ident(&capitalize(&attr.type_name[1..]));
+    database_url: &str,
-        tpe = quote_ty!(cx, Array<$subtype>);
+) -> Result<Conn, Box<MacResult>> where
-    } else {
+    Conn: Connection,
-        let type_name = str_to_ident(&capitalize(&attr.type_name));
+{
-        tpe = quote_ty!(cx, $type_name);
+    Conn::establish(database_url).map_err(|_| {
-    }
+        cx.span_err(sp, "failed to establish a database connection");
+        DummyResult::any(sp)
+    })
+}
 
-    if attr.nullable {
+
-        quote_ty!(cx, Nullable<$tpe>)
+// FIXME: Remove the duplicates of this function once expression level attributes
+// are stable (I believe this is in 1.7)
+#[cfg(all(feature = "sqlite", not(feature = "postgres")))]
+fn establish_connection(
+    cx: &mut ExtCtxt,
+    sp: Span,
+    database_url: &str,
+) -> Result<InferConnection, Box<MacResult>> {
+    establish_real_connection(cx, sp, database_url)
+}
+
+
+#[cfg(all(feature = "postgres", not(feature = "sqlite")))]
+fn establish_connection(
+    cx: &mut ExtCtxt,
+    sp: Span,
+    database_url: &str,
+) -> Result<InferConnection, Box<MacResult>> {
+    establish_real_connection(cx, sp, database_url)
+}
+
+#[cfg(all(feature = "sqlite", feature = "postgres"))]
+fn establish_connection(
+    cx: &mut ExtCtxt,
+    sp: Span,
+    database_url: &str,
+) -> Result<InferConnection, Box<MacResult>> {
+    if database_url.starts_with("postgres://") || database_url.starts_with("postgresql://") {
+        establish_real_connection(cx, sp, database_url).map(|c| InferConnection::Pg(c))
     } else {
-        tpe
+        establish_real_connection(cx, sp, database_url).map(|c| InferConnection::Sqlite(c))
     }
 }
 
-fn capitalize(name: &str) -> String {
+
-    name[..1].to_uppercase() + &name[1..]
+#[cfg(all(feature = "sqlite", not(feature = "postgres")))]
+fn get_table_data(conn: &InferConnection, table_name: &str)
+    -> QueryResult<Vec<ColumnInformation>>
+{
+    sqlite::get_table_data(conn, table_name)
 }
 
+#[cfg(all(feature = "postgres", not(feature = "sqlite")))]
+fn get_table_data(conn: &InferConnection, table_name: &str)
+    -> QueryResult<Vec<ColumnInformation>>
+{
+    pg::get_table_data(conn, table_name)
+}
+
+#[cfg(all(feature = "postgres", feature = "sqlite"))]
+fn get_table_data(conn: &InferConnection, table_name: &str)
+    -> QueryResult<Vec<ColumnInformation>>
+{
+    match *conn{
+        InferConnection::Sqlite(ref c) => sqlite::get_table_data(c, table_name),
+        InferConnection::Pg(ref c) => pg::get_table_data(c, table_name),
+    }
+}
+
+#[cfg(all(feature = "sqlite", not(feature = "postgres")))]
 fn load_table_names(
-    _cx: &mut ExtCtxt,
+    cx: &mut ExtCtxt,
-    _sp: Span,
+    sp: Span,
-    connection: &PgConnection,
+    connection: &InferConnection,
-) -> Result<Vec<String>, result::Error> {
+) -> Result<Vec<String>, ::diesel::result::Error> {
-    use diesel::prelude::*;
+    sqlite::load_table_names(cx, sp, connection)
-    use diesel::expression::dsl::sql;
+}
-
+
-    let query = select(sql::<types::VarChar>("table_name FROM information_schema.tables"))
+#[cfg(all(feature = "postgres", not(feature = "sqlite")))]
-        .filter(sql::<types::Bool>("table_schema = 'public' AND table_name NOT LIKE '\\_\\_%'"));
+fn load_table_names(
-    query.load(connection)
+    cx: &mut ExtCtxt,
+    sp: Span,
+    connection: &InferConnection,
+) -> Result<Vec<String>, ::diesel::result::Error> {
+    pg::load_table_names(cx, sp, connection)
+}
+
+#[cfg(all(feature = "sqlite", feature = "postgres"))]
+fn load_table_names(
+    cx: &mut ExtCtxt,
+    sp: Span,
+    connection: &InferConnection,
+) -> Result<Vec<String>, ::diesel::result::Error> {
+    match *connection {
+        InferConnection::Sqlite(ref c) => sqlite::load_table_names(cx, sp, c),
+        InferConnection::Pg(ref c) => pg::load_table_names(cx, sp, c),
+    }
+}
+
+#[cfg(all(feature = "sqlite", not(feature = "postgres")))]
+fn determine_column_type(cx: &mut ExtCtxt, attr: &ColumnInformation, _conn: &InferConnection)
+    -> P<ast::Ty>
+{
+    sqlite::determine_column_type(cx, attr)
+}
+
+#[cfg(all(feature = "postgres", not(feature = "sqlite")))]
+fn determine_column_type(cx: &mut ExtCtxt, attr: &ColumnInformation, _conn: &InferConnection)
+    -> P<ast::Ty>
+{
+    pg::determine_column_type(cx, attr)
+}
+
+#[cfg(all(feature = "sqlite", feature = "postgres"))]
+fn determine_column_type(cx: &mut ExtCtxt, attr: &ColumnInformation, conn: &InferConnection)
+    -> P<ast::Ty>
+{
+    match *conn {
+        InferConnection::Sqlite(_) => sqlite::determine_column_type(cx, attr),
+        InferConnection::Pg(_) => pg::determine_column_type(cx, attr),
+    }
 }

diesel_codegen/src/schema_inference/pg.rs

@@ -0,0 +1,90 @@
+use diesel::*;
+use diesel::pg::PgConnection;
+use syntax::ast;
+use syntax::codemap::Span;
+use syntax::ext::base::*;
+use syntax::ptr::P;
+use syntax::parse::token::str_to_ident;
+
+use super::data_structures::*;
+
+table! {
+    pg_attribute (attrelid) {
+        attrelid -> Oid,
+        attname -> VarChar,
+        atttypid -> Oid,
+        attnotnull -> Bool,
+        attnum -> SmallInt,
+        attisdropped -> Bool,
+    }
+}
+
+table! {
+    pg_type (oid) {
+        oid -> Oid,
+        typname -> VarChar,
+    }
+}
+
+joinable!(pg_attribute -> pg_type (atttypid));
+select_column_workaround!(pg_attribute -> pg_type (attrelid, attname, atttypid, attnotnull, attnum, attisdropped));
+select_column_workaround!(pg_type -> pg_attribute (oid, typname));
+
+table! {
+    pg_class (oid) {
+        oid -> Oid,
+        relname -> VarChar,
+    }
+}
+
+pub fn determine_column_type(cx: &mut ExtCtxt, attr: &ColumnInformation) -> P<ast::Ty> {
+    let tpe = if attr.type_name.starts_with("_") {
+        let subtype = str_to_ident(&capitalize(&attr.type_name[1..]));
+        quote_ty!(cx, Array<$subtype>)
+    } else {
+        let type_name = str_to_ident(&capitalize(&attr.type_name));
+        quote_ty!(cx, $type_name)
+    };
+
+    if attr.nullable {
+        quote_ty!(cx, Nullable<$tpe>)
+    } else {
+        tpe
+    }
+}
+
+fn capitalize(name: &str) -> String {
+    name[..1].to_uppercase() + &name[1..]
+}
+
+pub fn load_table_names(
+    _cx: &mut ExtCtxt,
+    _sp: Span,
+    connection: &PgConnection,
+) -> Result<Vec<String>, result::Error>
+{
+    use diesel::prelude::*;
+    use diesel::expression::dsl::sql;
+
+    let query = select(sql::<types::VarChar>("table_name FROM information_schema.tables"))
+        .filter(sql::<types::Bool>("table_schema = 'public' AND table_name NOT LIKE '\\_\\_%'"));
+    query.load(connection)
+}
+
+pub fn get_table_data(conn: &PgConnection, table_name: &str) -> QueryResult<Vec<ColumnInformation>> {
+    use self::pg_attribute::dsl::*;
+    use self::pg_type::dsl::{pg_type, typname};
+    let t_oid = try!(table_oid(conn, table_name));
+
+    pg_attribute.inner_join(pg_type)
+        .select((attname, typname, attnotnull))
+        .filter(attrelid.eq(t_oid))
+        .filter(attnum.gt(0).and(attisdropped.ne(true)))
+        .order(attnum)
+        .load(conn)
+}
+
+fn table_oid(conn: &PgConnection, table_name: &str) -> QueryResult<u32> {
+    use self::pg_class::dsl::*;
+    pg_class.select(oid).filter(relname.eq(table_name)).first(conn)
+}

diesel_codegen/src/schema_inference/sqlite.rs

@@ -0,0 +1,64 @@
+use diesel::*;
+use diesel::sqlite::SqliteConnection;
+use syntax::ast;
+use syntax::codemap::Span;
+use syntax::ext::base::*;
+use syntax::ptr::P;
+
+use super::data_structures::*;
+
+table!{
+    pragma_table_info (cid){
+        cid ->Integer,
+        name -> VarChar,
+        type_name -> VarChar,
+        notnull -> Bool,
+        dflt_value -> Nullable<VarChar>,
+        pk -> Integer,
+    }
+}
+
+pub fn get_table_data(conn: &SqliteConnection, table_name: &str)
+    -> QueryResult<Vec<ColumnInformation>>
+{
+    conn.execute_pragma::<pragma_table_info::SqlType, ColumnInformation>(
+        &format!("PRAGMA TABLE_INFO('{}')", table_name))
+}
+
+fn is_text(type_name: &str) -> bool {
+    type_name.contains("char")||
+        type_name.contains("clob")||
+        type_name.contains("text")
+}
+
+pub fn determine_column_type(cx: &mut ExtCtxt, attr: &ColumnInformation) -> P<ast::Ty> {
+    let type_name=attr.type_name.to_lowercase();
+    let tpe = if type_name.contains("int") {
+        quote_ty!(cx, ::diesel::types::BigInteger)
+    } else if is_text(&type_name) {
+        quote_ty!(cx, ::diesel::types::Text)
+    } else if type_name.contains("blob") || type_name.is_empty() {
+        quote_ty!(cx, ::diesel::types::Binary)
+    } else {
+        quote_ty!(cx, ::diesel::types::Double)
+    };
+
+    if attr.nullable {
+        quote_ty!(cx, Nullable<$tpe>)
+    } else {
+        tpe
+    }
+}
+
+pub fn load_table_names(
+    _cx: &mut ExtCtxt,
+    _sp: Span,
+    connection: &SqliteConnection,
+) -> Result<Vec<String>, result::Error> {
+    use diesel::prelude::*;
+    use diesel::expression::dsl::sql;
+
+    let query = select(sql::<types::VarChar>("name FROM sqlite_master"))
+        .filter(sql::<types::Bool>("type='table' AND name NOT LIKE '\\_\\_%'"));
+    query.load(connection)
+}

diesel_tests/Cargo.toml

@@ -24,6 +24,7 @@ quickcheck = "0.2.25"
 [features]
 default = ["syntex", "diesel_codegen/with-syntex", "dotenv_codegen"]
 unstable = ["diesel_codegen/nightly", "diesel/unstable", "dotenv_macros"]
+syntex = ["diesel_codegen/with-syntex"]
 postgres = ["diesel/postgres", "diesel_codegen/postgres"]
 sqlite = ["diesel/sqlite", "diesel_codegen/sqlite"]