Open Stats : LoadPSVFilesToDB

HomePage :: Categories :: PageIndex :: RecentChanges :: RecentlyCommented :: Login/Register

LoadPSVFilesToDB


This is a small script I created the load the PSV files created with CapNKipe into MySQL, PostgreSQL, and Sqlite. It requires you have TCL or TCLKIT installed on your system.


get-capnkipe-loaded.tcl
#!/usr/bin/tclsh

array set cfg {}
set db_types {}
set valid_types [list pgsql mysql sqlite3]

set cfg(pgsql_bin) "/usr/bin/psql"
set cfg(pgsql_args) {retrosheet}

set cfg(mysql_bin) "/usr/bin/mysql"
set cfg(mysql_args) {-uroot retrosheet}

set cfg(sqlite3_bin) "/home/mek/bin/sqlite3"
set cfg(sqlite3_args) {/home/mek/retrosheet/db/retrosheet.db}

proc run { command } {
    if {[catch {eval $command} err]} {
        foreach line [split $err "\n"] {
            if {[string length $line] > 0} {
                puts stderr $line
            }
        }
    }
}

proc db_cmd {bin args} {
    set c exec
    lappend c $bin
    foreach arg $args {
        lappend c $arg
    }
    return $c
}

proc create_db { cfg db_types } {
   
    upvar $cfg cf

    foreach db $db_types {
        set db_exec "[db_cmd $cf(${db}_bin) $cf(${db}_args)]"
        set cmd "$db_exec < $db-create-database.sql"
        puts "Creating tables for $db ... "
        run $cmd
    }
}

foreach db $argv {
  if {[lsearch -exact $valid_types $db] != -1} {
    lappend db_types $db
    create_db cfg $db_types
  } else {
    puts stderr "Type $db invalid, skipping."
  }
}


for {set year 1957} {$year <= 2006} {incr year} {
    foreach type {games events teams rosters} {
        foreach ext {psv} {
            set f [file join [pwd ] ${year}-${type}.${ext}]
            if [file exists $f] {
                foreach db $db_types {
                    set db_exec "[db_cmd $cfg(${db}_bin) $cfg(${db}_args)]"
                    switch [string toupper $db] {
                        PGSQL {
                            set cmd "$db_exec << \"copy $type from \'$f\' delimiters \'|\'csv;\"\n"
                        }
                        MYSQL {
                            set cmd "$db_exec << \"load data infile \'$f\' into table $type fields terminated by \'|\';\"\n"
                        }
                        SQLITE3 {
                            set cmd "$db_exec << \".import \'$f\' $type\"\n"
                        }
                    }
                    puts "Loading $year $type files into $db database ... "
                    run $cmd
                }
            }
        }
    }
}


For example to load files into PostgreSQL, run:

mek@barker:~/retrosheet/csv$ ../tclkit get-capnkipe-loaded.tcl pgsql


Categories

CategorySoftware

There are no comments on this page. [Add comment]

Valid XHTML 1.0 Transitional :: Valid CSS :: Powered by Wikka Wakka Wiki 1.1.6.2
Page was generated in 0.2937 seconds