From bbfbdd845016e60f7c6326462aafb7b76c2b9b14 Mon Sep 17 00:00:00 2001 From: Corby Krick Date: Mon, 4 Mar 2013 22:51:50 -0600 Subject: [PATCH 1/8] Adding .TQ8 file types. --- application/config/mimes.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/application/config/mimes.php b/application/config/mimes.php index efd4dc78..856a7b8d 100644 --- a/application/config/mimes.php +++ b/application/config/mimes.php @@ -101,8 +101,10 @@ $mimes = array( 'hqx' => 'application/mac-binhex40', 'json' => array('application/json', 'text/json'), 'adi' => 'application/octet-stream', 'ADI' => 'application/octet-stream', + 'tq8' => 'application/octet-stream', + 'TQ8' => 'application/octet-stream', ); /* End of file mimes.php */ -/* Location: ./application/config/mimes.php */ \ No newline at end of file +/* Location: ./application/config/mimes.php */ From cbe66352717ce7daeee3c9af4a76b162fa6f7bb1 Mon Sep 17 00:00:00 2001 From: Corby Krick Date: Mon, 4 Mar 2013 22:57:29 -0600 Subject: [PATCH 2/8] Added export_lotw to export an ADIF file of QSOs that have not been uploaded to LoTW --- application/controllers/adif.php | 12 ++++++++++++ application/models/adif_data.php | 8 ++++++++ 2 files changed, 20 insertions(+) diff --git a/application/controllers/adif.php b/application/controllers/adif.php index 730e9548..bdc557a6 100644 --- a/application/controllers/adif.php +++ b/application/controllers/adif.php @@ -49,6 +49,18 @@ class adif extends CI_Controller { } + public function export_lotw() + { + // Set memory limit to unlimited to allow heavy usage + ini_set('memory_limit', '-1'); + + $this->load->model('adif_data'); + + $data['qsos'] = $this->adif_data->export_lotw(); + + $this->load->view('adif/data/exportall', $data); + } + public function import() { $data['page_title'] = "ADIF Import"; diff --git a/application/models/adif_data.php b/application/models/adif_data.php index ed5b51c7..564403f0 100644 --- a/application/models/adif_data.php +++ b/application/models/adif_data.php @@ -23,6 +23,14 @@ class adif_data extends CI_Model { return $query; } + + function export_lotw() { + $this->db->where("COL_LOTW_QSL_SENT != 'Y'"); + $this->db->order_by("COL_TIME_ON", "ASC"); + $query = $this->db->get($this->config->item('table_name')); + + return $query; + } } ?> From bc17566e49e96d3b4c15cd0e5a22f96368a4238a Mon Sep 17 00:00:00 2001 From: Corby Krick Date: Mon, 4 Mar 2013 22:59:33 -0600 Subject: [PATCH 3/8] Added a view for uploading TQ8 (LoTW signed files). Working on the controller for functionality yet. --- application/controllers/lotw.php | 39 +++++++++++++++++++++++++++++ application/views/layout/header.php | 1 + application/views/lotw/export.php | 24 ++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 application/views/lotw/export.php diff --git a/application/controllers/lotw.php b/application/controllers/lotw.php index b9eefd75..0fb745fc 100644 --- a/application/controllers/lotw.php +++ b/application/controllers/lotw.php @@ -167,4 +167,43 @@ class Lotw extends CI_Controller { } } } // end function + + public function export() { + $data['page_title'] = "LoTW .TQ8 Upload"; + + $config['upload_path'] = './uploads/'; + $config['allowed_types'] = 'tq8|TQ8'; + + $this->load->library('upload', $config); + + if ( ! $this->upload->do_upload()) + { + $data['error'] = $this->upload->display_errors(); + + $this->load->view('layout/header', $data); + $this->load->view('lotw/export'); + $this->load->view('layout/footer'); + } + else + { + + $data = array('upload_data' => $this->upload->data()); + + // Curl stuff goes here + + //unlink('./uploads/'.$data['upload_data']['file_name']); + + $data['page_title'] = "LoTW .TQ8 Sent"; + $this->load->view('layout/header', $data); + + //Perhaps return some sort of success page + $this->load->view('lotw/analysis'); + $this->load->view('layout/footer'); + + + } + + + } + } // end class \ No newline at end of file diff --git a/application/views/layout/header.php b/application/views/layout/header.php index 170c7b30..d2e0fc56 100644 --- a/application/views/layout/header.php +++ b/application/views/layout/header.php @@ -81,6 +81,7 @@
  • Data Export
  • API
  • LoTW Import
    • +
  • LoTW Export
    • diff --git a/application/views/lotw/export.php b/application/views/lotw/export.php new file mode 100644 index 00000000..32bb708c --- /dev/null +++ b/application/views/lotw/export.php @@ -0,0 +1,24 @@ +
    +

    + + +

    Step 1

    + Export an ADIF file of QSOs that have not been uploaded to LoTW. + +

    Step 2

    +

    Use Trusted QSL to sign the exported file.

    + +

    Step 3

    +

    Select the signed file and click "Upload". It will be sent to LoTW for processing.

    + + +

    Important Log files must have the file type .tq8

    + + +

    Cloudlog will use the LoTW username an password stored in your user profile to download a report from LoTW for you. The report Cloudlog downloads will have all confirmations since your last LoTW confirmation, up until now.

    + + + + + +
    From 4a96fbe55a3237759e3e07eca582d3eb4e6e79e4 Mon Sep 17 00:00:00 2001 From: Corby Krick Date: Mon, 4 Mar 2013 23:01:45 -0600 Subject: [PATCH 4/8] Adding uploads/*.ADI, uploads/*.tq8 and uploads/*.TQ8 to ignore list. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 3f757471..5a957682 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ /application/config/database.php /application/config/config.php /uploads/*.adi +/uploads/*.ADI +/uploads/*.tq8 +/uploads/*.TQ8 From d014e2431762b4cbe538bfbbb50fedee413592e0 Mon Sep 17 00:00:00 2001 From: Corby Krick Date: Tue, 5 Mar 2013 19:19:52 -0600 Subject: [PATCH 5/8] Working prototype of LoTW file upload. Needs a bit of polish yet. --- application/config/migration.php | 2 +- application/controllers/lotw.php | 81 +++++++++++++++++-- .../migrations/003_add_lotw_login_url.php | 21 +++++ 3 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 application/migrations/003_add_lotw_login_url.php diff --git a/application/config/migration.php b/application/config/migration.php index 93759a4f..d7092f91 100644 --- a/application/config/migration.php +++ b/application/config/migration.php @@ -21,7 +21,7 @@ $config['migration_enabled'] = TRUE; | be upgraded / downgraded to. | */ -$config['migration_version'] = 2; +$config['migration_version'] = 3; /* diff --git a/application/controllers/lotw.php b/application/controllers/lotw.php index 0fb745fc..14b67db5 100644 --- a/application/controllers/lotw.php +++ b/application/controllers/lotw.php @@ -123,9 +123,6 @@ class Lotw extends CI_Controller { // Query the logbook to determine when the last LoTW confirmation was $lotw_last_qsl_date = $this->logbook_model->lotw_last_qsl_date(); - - // TODO: Consolidate code - // TODO: Specifiy in config file whether we want LoTW confirms as V or Y. Both are acceptable under ADIF specification. HRD seems to use V. Everyone else that I've used uses Y. // Build URL for LoTW report file $lotw_url .= "?"; @@ -189,15 +186,89 @@ class Lotw extends CI_Controller { $data = array('upload_data' => $this->upload->data()); + // Figure out how we should be marking QSLs confirmed via LoTW + $query = $query = $this->db->query('SELECT lotw_login_url FROM config'); + $q = $query->row(); + $config['lotw_login_url'] = $q->lotw_login_url; + + $query = $this->user_model->get_by_id($this->session->userdata('user_id')); + $q = $query->row(); + $config['user_lotw_name'] = $q->user_lotw_name; + $config['user_lotw_password'] = $q->user_lotw_password; + // Curl stuff goes here - //unlink('./uploads/'.$data['upload_data']['file_name']); + // First we need to get a cookie + + // options + $LOGIN = $config['user_lotw_name']; + $PASSWORD = $config['user_lotw_password']; + $cookie_file_path = "./uploads/cookies.txt"; + $LOGINURL = $config['lotw_login_url']; + $agent = "Mozilla/4.0 (compatible;)"; + + + // begin script + $ch = curl_init(); + + // extra headers + $headers[] = "Accept: */*"; + $headers[] = "Connection: Keep-Alive"; + + // basic curl options for all requests + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + curl_setopt($ch, CURLOPT_HEADER, 0); + + // TODO: These SSL things should probably be set to true :) + curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0); + curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); + curl_setopt($ch, CURLOPT_USERAGENT, $agent); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1); + curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie_file_path); + curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie_file_path); + + // Set login URL + curl_setopt($ch, CURLOPT_URL, $LOGINURL); + + $fields['login'] = $LOGIN; + $fields['password'] = $PASSWORD; + $fields['acct_sel'] = ""; + + // set postfields using what we extracted from the form + $POSTFIELDS = http_build_query($fields); + + // set post options + curl_setopt($ch, CURLOPT_POST, 1); + curl_setopt($ch, CURLOPT_POSTFIELDS, $POSTFIELDS); + + // perform login + // TODO: probably should look at result at some point to verify that our login worked. + $result = curl_exec($ch); + + // Now we need to use that cookie and upload the file + // change URL to upload destination URL + curl_setopt($ch, CURLOPT_URL, $config['lotw_login_url']); + + // Grab the file + $postfile = array( + "upfile"=>"@./uploads/".$data['upload_data']['file_name'], + ); + + //Upload it + //TODO: Read the output + curl_setopt($ch, CURLOPT_POSTFIELDS, $postfile); + $response = curl_exec($ch); + + // Now we need to clean up + unlink($cookie_file_path); + unlink('./uploads/'.$data['upload_data']['file_name']); $data['page_title'] = "LoTW .TQ8 Sent"; $this->load->view('layout/header', $data); //Perhaps return some sort of success page - $this->load->view('lotw/analysis'); + $this->load->view('lotw/export'); $this->load->view('layout/footer'); diff --git a/application/migrations/003_add_lotw_login_url.php b/application/migrations/003_add_lotw_login_url.php new file mode 100644 index 00000000..c3ad57f0 --- /dev/null +++ b/application/migrations/003_add_lotw_login_url.php @@ -0,0 +1,21 @@ +dbforge->add_column('config', $fields); + + $sql = "UPDATE config SET lotw_login_url = 'https://p1k.arrl.org/lotwuser/default' WHERE id=1"; + + $this->db->query($sql); + } + + public function down() + { + $this->dbforge->drop_column('config', 'lotw_login_url'); + } +} +?> From 048891a69c08ae0ab17508007d7588908e3f763f Mon Sep 17 00:00:00 2001 From: Corby Krick Date: Wed, 6 Mar 2013 23:31:19 -0600 Subject: [PATCH 6/8] Added method for marking QSOs exported for LoTW as sent. --- application/controllers/adif.php | 5 +++++ application/models/adif_data.php | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/application/controllers/adif.php b/application/controllers/adif.php index bdc557a6..197a6f40 100644 --- a/application/controllers/adif.php +++ b/application/controllers/adif.php @@ -59,6 +59,11 @@ class adif extends CI_Controller { $data['qsos'] = $this->adif_data->export_lotw(); $this->load->view('adif/data/exportall', $data); + + foreach ($data['qsos']->result() as $qso) + { + $this->adif_data->mark_lotw_sent($qso->COL_PRIMARY_KEY); + } } public function import() { diff --git a/application/models/adif_data.php b/application/models/adif_data.php index 564403f0..2d1f2da7 100644 --- a/application/models/adif_data.php +++ b/application/models/adif_data.php @@ -31,6 +31,15 @@ class adif_data extends CI_Model { return $query; } + + function mark_lotw_sent($id) { + $data = array( + 'COL_LOTW_QSL_SENT' => 'Y' + ); + + $this->db->where('COL_PRIMARY_KEY', $id); + $this->db->update($this->config->item('table_name'), $data); + } } ?> From f3ac92a305142467a6f9a4ce120bf7fe98a76d84 Mon Sep 17 00:00:00 2001 From: Corby Krick Date: Wed, 6 Mar 2013 23:33:39 -0600 Subject: [PATCH 7/8] Adding support for LoTW export. --- application/controllers/lotw.php | 65 ++++++++++++++++++------------- application/views/lotw/export.php | 7 +++- application/views/lotw/status.php | 15 +++++++ 3 files changed, 59 insertions(+), 28 deletions(-) create mode 100644 application/views/lotw/status.php diff --git a/application/controllers/lotw.php b/application/controllers/lotw.php index 14b67db5..aa788875 100644 --- a/application/controllers/lotw.php +++ b/application/controllers/lotw.php @@ -118,7 +118,7 @@ class Lotw extends CI_Controller { // TODO: We don't actually see the error message if ($data['user_lotw_name'] == '' || $data['user_lotw_password'] == '') { - $this->session->set_flashdata('warning', 'You have not defined your ARRL LoTW credentials!'); redirect('dashboard'); + $this->session->set_flashdata('warning', 'You have not defined your ARRL LoTW credentials!'); redirect('lotw/import'); } // Query the logbook to determine when the last LoTW confirmation was @@ -182,8 +182,7 @@ class Lotw extends CI_Controller { $this->load->view('layout/footer'); } else - { - + { $data = array('upload_data' => $this->upload->data()); // Figure out how we should be marking QSLs confirmed via LoTW @@ -191,23 +190,26 @@ class Lotw extends CI_Controller { $q = $query->row(); $config['lotw_login_url'] = $q->lotw_login_url; + // Set some fields that we're going to need for ARRL login $query = $this->user_model->get_by_id($this->session->userdata('user_id')); $q = $query->row(); - $config['user_lotw_name'] = $q->user_lotw_name; - $config['user_lotw_password'] = $q->user_lotw_password; + $fields['login'] = $q->user_lotw_name; + $fields['password'] = $q->user_lotw_password; + $fields['acct_sel'] = ""; + if ($fields['login'] == '' || $fields['password'] == '') + { + $this->session->set_flashdata('warning', 'You have not defined your ARRL LoTW credentials!'); redirect('lotw/status'); + } + // Curl stuff goes here // First we need to get a cookie // options - $LOGIN = $config['user_lotw_name']; - $PASSWORD = $config['user_lotw_password']; $cookie_file_path = "./uploads/cookies.txt"; - $LOGINURL = $config['lotw_login_url']; $agent = "Mozilla/4.0 (compatible;)"; - // begin script $ch = curl_init(); @@ -229,11 +231,7 @@ class Lotw extends CI_Controller { curl_setopt($ch, CURLOPT_COOKIEJAR, $cookie_file_path); // Set login URL - curl_setopt($ch, CURLOPT_URL, $LOGINURL); - - $fields['login'] = $LOGIN; - $fields['password'] = $PASSWORD; - $fields['acct_sel'] = ""; + curl_setopt($ch, CURLOPT_URL, $config['lotw_login_url']); // set postfields using what we extracted from the form $POSTFIELDS = http_build_query($fields); @@ -243,8 +241,12 @@ class Lotw extends CI_Controller { curl_setopt($ch, CURLOPT_POSTFIELDS, $POSTFIELDS); // perform login - // TODO: probably should look at result at some point to verify that our login worked. $result = curl_exec($ch); + if (stristr($result, "Username/password incorrect")) + { + $this->session->set_flashdata('warning', 'Your ARRL username and/or password is incorrect.'); redirect('lotw/status'); + } + // Now we need to use that cookie and upload the file // change URL to upload destination URL @@ -256,25 +258,34 @@ class Lotw extends CI_Controller { ); //Upload it - //TODO: Read the output curl_setopt($ch, CURLOPT_POSTFIELDS, $postfile); $response = curl_exec($ch); - + if (stristr($response, "accepted")) + { + $this->session->set_flashdata('lotw_status', 'accepted'); + $data['page_title'] = "LoTW .TQ8 Sent"; + } + elseif (stristr($response, "rejected")) + { + $this->session->set_flashdata('lotw_status', 'rejected'); + $data['page_title'] = "LoTW .TQ8 Sent"; + } + else + { + // If we're here, we didn't find what we're looking for in the ARRL response + // and LoTW is probably down or broken. + $this->session->set_flashdata('warning', 'Did not receive proper response from LoTW. Try again later.'); + $data['page_title'] = "LoTW .TQ8 Not Sent"; + } + // Now we need to clean up unlink($cookie_file_path); unlink('./uploads/'.$data['upload_data']['file_name']); - - $data['page_title'] = "LoTW .TQ8 Sent"; - $this->load->view('layout/header', $data); - //Perhaps return some sort of success page - $this->load->view('lotw/export'); + $this->load->view('layout/header', $data); + $this->load->view('lotw/status'); $this->load->view('layout/footer'); - - - } - - + } } } // end class \ No newline at end of file diff --git a/application/views/lotw/export.php b/application/views/lotw/export.php index 32bb708c..1f755828 100644 --- a/application/views/lotw/export.php +++ b/application/views/lotw/export.php @@ -1,7 +1,12 @@

    - + session->flashdata('warning')) { ?> +
    + session->flashdata('warning'); ?> +
    + +

    Step 1

    Export an ADIF file of QSOs that have not been uploaded to LoTW. diff --git a/application/views/lotw/status.php b/application/views/lotw/status.php new file mode 100644 index 00000000..fb7fee8f --- /dev/null +++ b/application/views/lotw/status.php @@ -0,0 +1,15 @@ +
    +

    + +session->flashdata('warning')) { ?> +
    + session->flashdata('warning'); ?> +
    + +session->flashdata('lotw_status')) { ?> +
    + session->flashdata('lotw_status'); ?> +
    + + +
    From 83f6eb41db32b9f1268a571ff8ec66a059cf0cc3 Mon Sep 17 00:00:00 2001 From: Corby Krick Date: Fri, 8 Mar 2013 18:25:47 -0600 Subject: [PATCH 8/8] Now populating COL_LOTW_QSLSDATE --- application/models/adif_data.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/application/models/adif_data.php b/application/models/adif_data.php index 2d1f2da7..8e91e6ad 100644 --- a/application/models/adif_data.php +++ b/application/models/adif_data.php @@ -36,7 +36,8 @@ class adif_data extends CI_Model { $data = array( 'COL_LOTW_QSL_SENT' => 'Y' ); - + + $this->db->set('COL_LOTW_QSLSDATE', 'CURDATE()', FALSE); $this->db->where('COL_PRIMARY_KEY', $id); $this->db->update($this->config->item('table_name'), $data); }