mirror
/
Cloudlog
kopia lustrzana https://github.com/magicbug/Cloudlog
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Optimize DXCC list queries and add DB indexes 

Refactored Workabledxcc controller and model to batch DXCC entity and worked/confirmed status lookups for improved performance. Added migration to create composite indexes on DXCC-related columns to further speed up queries. Updated migration version to 205.
pull/3320/head
Peter Goodhall 2025-08-11 14:50:56 +01:00
rodzic 61f5594598
commit 9b17ff9250
4 zmienionych plików z 275 dodań i 101 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

2
application/config/migration.php
Wyświetl plik

@ -22,7 +22,7 @@ $config['migration_enabled'] = TRUE;
|
*/
$config['migration_version'] = 204;
$config['migration_version'] = 205;
/*
|--------------------------------------------------------------------------

61
application/controllers/Workabledxcc.php
Wyświetl plik

@ -29,53 +29,47 @@ class Workabledxcc extends CI_Controller
public function dxcclist()
{
$json = file_get_contents($this->optionslib->get_option('dxped_url'));
// Decode the JSON data into a PHP array
$dataResult = json_decode($json, true);
// Initialize an empty array to store the required data
$requiredData = array();
if (empty($dataResult)) {
$data['dxcclist'] = array();
$this->load->view('/workabledxcc/components/dxcclist', $data);
return;
}
// Get Date format
if ($this->session->userdata('user_date_format')) {
// If Logged in and session exists
$custom_date_format = $this->session->userdata('user_date_format');
} else {
// Get Default date format from /config/cloudlog.php
$custom_date_format = $this->config->item('qso_date_format');
}
// Iterate through the decoded JSON data
foreach ($dataResult as $item) {
// Create a new array with the required fields and add it to the main array
$oldStartDate = DateTime::createFromFormat('Y-m-d', $item['0']);
// Load models once
$this->load->model('logbook_model');
$this->load->model('Workabledxcc_model');
// Get all DXCC entities for all callsigns in one batch
$callsigns = array_column($dataResult, 'callsign');
$dates = array_column($dataResult, '0');
$dxccEntities = $this->Workabledxcc_model->batchDxccLookup($callsigns, $dates);
// Get worked/confirmed status for all entities in batch
$uniqueEntities = array_unique(array_filter($dxccEntities));
$dxccStatus = $this->Workabledxcc_model->batchDxccWorkedStatus($uniqueEntities);
// Process results
$requiredData = array();
foreach ($dataResult as $index => $item) {
$oldStartDate = DateTime::createFromFormat('Y-m-d', $item['0']);
$StartDate = $oldStartDate->format($custom_date_format);
$oldEndDate = DateTime::createFromFormat('Y-m-d', $item['1']);
$EndDate = $oldEndDate->format($custom_date_format);
$oldStartDate1 = DateTime::createFromFormat('Y-m-d', $item['0']);
$StartDate1 = $oldStartDate1->format('Y-m-d');
$this->load->model('logbook_model');
$dxccInfo = $this->logbook_model->dxcc_lookup($item['callsign'], $StartDate1);
// Call DXCC Worked function to check if the DXCC has been worked before
if (isset($dxccInfo['entity'])) {
$dxccWorked = $this->dxccWorked($dxccInfo['entity']);
} else {
// Handle the case where 'entity' is not set in $dxccInfo
$dxccWorked = array(
'workedBefore' => false,
'confirmed' => false,
);
}
// Get DXCC status for this callsign
$entity = $dxccEntities[$index] ?? null;
$worked = $entity && isset($dxccStatus[$entity]) ? $dxccStatus[$entity] : ['workedBefore' => false, 'confirmed' => false];
$requiredData[] = array(
'clean_date' => $item['0'],
@ -84,15 +78,12 @@ class Workabledxcc extends CI_Controller
'country' => $item['2'],
'notes' => $item['6'],
'callsign' => $item['callsign'],
'workedBefore' => $dxccWorked['workedBefore'],
'confirmed' => $dxccWorked['confirmed'],
'workedBefore' => $worked['workedBefore'],
'confirmed' => $worked['confirmed'],
);
}
$data['dxcclist'] = $requiredData;
// Return the array with the required data
$this->load->view('/workabledxcc/components/dxcclist', $data);
}

49
application/migrations/205_add_workable_dxcc_indexes.php 100644
Wyświetl plik

@ -0,0 +1,49 @@
<?php
defined('BASEPATH') OR exit('No direct script access allowed');
class Migration_add_workable_dxcc_indexes extends CI_Migration {
public function up()
{
// Add composite index for workable DXCC queries
// This will greatly improve performance for COL_COUNTRY + station_id + COL_PROP_MODE queries
$this->db->db_debug = false;
// Check if index already exists
$index_exists = $this->db->query("SHOW INDEX FROM ".$this->config->item('table_name')." WHERE Key_name = 'idx_workable_dxcc'")->num_rows();
if ($index_exists == 0) {
$sql = "ALTER TABLE ".$this->config->item('table_name')." ADD INDEX `idx_workable_dxcc` (`COL_COUNTRY`, `station_id`, `COL_PROP_MODE`)";
$this->db->query($sql);
}
// Add index for confirmation status columns
$conf_index_exists = $this->db->query("SHOW INDEX FROM ".$this->config->item('table_name')." WHERE Key_name = 'idx_qsl_confirmations'")->num_rows();
if ($conf_index_exists == 0) {
$sql = "ALTER TABLE ".$this->config->item('table_name')." ADD INDEX `idx_qsl_confirmations` (`COL_QSL_RCVD`, `COL_LOTW_QSL_RCVD`, `COL_EQSL_QSL_RCVD`, `COL_QRZCOM_QSO_DOWNLOAD_STATUS`)";
$this->db->query($sql);
}
$this->db->db_debug = true;
}
public function down()
{
$this->db->db_debug = false;
// Drop the indexes if they exist
$index_exists = $this->db->query("SHOW INDEX FROM ".$this->config->item('table_name')." WHERE Key_name = 'idx_workable_dxcc'")->num_rows();
if ($index_exists > 0) {
$this->db->query("ALTER TABLE ".$this->config->item('table_name')." DROP INDEX `idx_workable_dxcc`");
}
$conf_index_exists = $this->db->query("SHOW INDEX FROM ".$this->config->item('table_name')." WHERE Key_name = 'idx_qsl_confirmations'")->num_rows();
if ($conf_index_exists > 0) {
$this->db->query("ALTER TABLE ".$this->config->item('table_name')." DROP INDEX `idx_qsl_confirmations`");
}
$this->db->db_debug = true;
}
}

264
application/models/Workabledxcc_model.php
Wyświetl plik

@ -2,87 +2,221 @@
class Workabledxcc_model extends CI_Model
{
// Cache for DXCC lookups to avoid repeated queries
private $dxccCache = array();
private $workedCache = array();
/**
* Batch DXCC lookup for multiple callsigns
* @param array $callsigns Array of callsigns
* @param array $dates Array of dates corresponding to callsigns
* @return array Array of DXCC entities indexed by callsign index
*/
public function batchDxccLookup($callsigns, $dates)
{
$this->load->model('logbook_model');
$entities = array();
foreach ($callsigns as $index => $callsign) {
$cacheKey = $callsign . '_' . $dates[$index];
if (!isset($this->dxccCache[$cacheKey])) {
$dxccInfo = $this->logbook_model->dxcc_lookup($callsign, $dates[$index]);
$this->dxccCache[$cacheKey] = isset($dxccInfo['entity']) ? $dxccInfo['entity'] : null;
}
$entities[$index] = $this->dxccCache[$cacheKey];
}
return $entities;
}
/**
* Batch check if DXCC entities have been worked/confirmed
* @param array $entities Array of unique DXCC entities
* @return array Array of worked/confirmed status indexed by entity
*/
public function batchDxccWorkedStatus($entities)
{
if (empty($entities)) {
return array();
}
$user_default_confirmation = $this->session->userdata('user_default_confirmation');
$this->load->model('logbooks_model');
$logbooks_locations_array = $this->logbooks_model->list_logbook_relationships($this->session->userdata('active_station_logbook'));
if (empty($logbooks_locations_array)) {
return array_fill_keys($entities, ['workedBefore' => false, 'confirmed' => false]);
}
$results = array();
// Build confirmation criteria once
$confirmationCriteria = $this->buildConfirmationCriteria($user_default_confirmation);
// Batch query for worked status
$workedResults = $this->batchWorkedQuery($entities, $logbooks_locations_array);
// Batch query for confirmed status
$confirmedResults = $this->batchConfirmedQuery($entities, $logbooks_locations_array, $confirmationCriteria);
// Combine results
foreach ($entities as $entity) {
$results[$entity] = [
'workedBefore' => isset($workedResults[$entity]),
'confirmed' => isset($confirmedResults[$entity])
];
}
return $results;
}
/**
* Build confirmation criteria SQL based on user preferences
*/
private function buildConfirmationCriteria($user_default_confirmation)
{
$criteria = array();
if (isset($user_default_confirmation) && strpos($user_default_confirmation, 'Q') !== false) {
$criteria[] = "COL_QSL_RCVD='Y'";
}
if (isset($user_default_confirmation) && strpos($user_default_confirmation, 'L') !== false) {
$criteria[] = "COL_LOTW_QSL_RCVD='Y'";
}
if (isset($user_default_confirmation) && strpos($user_default_confirmation, 'E') !== false) {
$criteria[] = "COL_EQSL_QSL_RCVD='Y'";
}
if (isset($user_default_confirmation) && strpos($user_default_confirmation, 'Z') !== false) {
$criteria[] = "COL_QRZCOM_QSO_DOWNLOAD_STATUS='Y'";
}
return empty($criteria) ? '1=0' : '(' . implode(' OR ', $criteria) . ')';
}
/**
* Batch query to check which entities have been worked
*/
private function batchWorkedQuery($entities, $logbooks_locations_array)
{
// Use a single query with GROUP BY to check all entities at once
$this->db->select('COL_COUNTRY')
->distinct()
->from($this->config->item('table_name'))
->where('COL_PROP_MODE !=', 'SAT')
->where_in('station_id', $logbooks_locations_array)
->where_in('COL_COUNTRY', array_map('urlencode', $entities));
$query = $this->db->get();
$results = array();
foreach ($query->result() as $row) {
$results[urldecode($row->COL_COUNTRY)] = true;
}
return $results;
}
/**
* Batch query to check which entities have been confirmed
*/
private function batchConfirmedQuery($entities, $logbooks_locations_array, $confirmationCriteria)
{
if ($confirmationCriteria === '1=0') {
return array();
}
$this->db->select('COL_COUNTRY')
->distinct()
->from($this->config->item('table_name'))
->where('COL_PROP_MODE !=', 'SAT')
->where($confirmationCriteria)
->where_in('station_id', $logbooks_locations_array)
->where_in('COL_COUNTRY', array_map('urlencode', $entities));
$query = $this->db->get();
$results = array();
foreach ($query->result() as $row) {
$results[urldecode($row->COL_COUNTRY)] = true;
}
return $results;
}
public function GetThisWeek()
{
$json = file_get_contents($this->optionslib->get_option('dxped_url'));
// Step 2: Convert the JSON data to an array.
$json = file_get_contents($this->optionslib->get_option('dxped_url'));
$data = json_decode($json, true);
// Step 3: Create a new array to hold the records for this week.
$thisWeekRecords = [];
if (empty($data)) {
return array();
}
// Get the start and end of this week.
$thisWeekRecords = [];
$startOfWeek = (new DateTime())->setISODate((new DateTime())->format('o'), (new DateTime())->format('W'), 1);
$endOfWeek = (clone $startOfWeek)->modify('+6 days');
// Step 4: Iterate over the array.
foreach ($data as $record) {
// Get Date format
if ($this->session->userdata('user_date_format')) {
$custom_date_format = $this->session->userdata('user_date_format');
} else {
$custom_date_format = $this->config->item('qso_date_format');
}
// Convert "0" and "1" to DateTime objects.
// First pass: filter records for this week
$weekRecords = array();
foreach ($data as $record) {
$startDate = new DateTime($record['0']);
$endDate = new DateTime($record['1']);
// Step 5: Check if the start date or end date is within this week.
if (($startDate >= $startOfWeek && $startDate <= $endOfWeek) || ($endDate >= $startOfWeek && $endDate <= $endOfWeek)) {
$endDate = new DateTime($record['1']);
$now = new DateTime();
$interval = $now->diff($endDate);
$daysLeft = $interval->days;
// If daysLeft is 0, set it to "Last day"
if ($daysLeft == 0) {
$daysLeft = "Last day";
} else {
$daysLeft = $daysLeft . " days left";
}
// Add daysLeft to record
$record['daysLeft'] = $daysLeft;
// Get Date format
if ($this->session->userdata('user_date_format')) {
// If Logged in and session exists
$custom_date_format = $this->session->userdata('user_date_format');
} else {
// Get Default date format from /config/cloudlog.php
$custom_date_format = $this->config->item('qso_date_format');
}
// Create a new array with the required fields and add it to the main array
$oldStartDate = DateTime::createFromFormat('Y-m-d', $record['0']);
$StartDate = $oldStartDate->format($custom_date_format);
$record['startDate'] = $StartDate;
$oldEndDate = DateTime::createFromFormat('Y-m-d', $record['1']);
$EndDate = $oldEndDate->format($custom_date_format);
$record['endDate'] = $EndDate;
$record['confirmed'] = true; // or false, depending on your logic
$CI = &get_instance();
$CI->load->model('logbook_model');
$dxccInfo = $CI->logbook_model->dxcc_lookup($record['callsign'], $startDate->format('Y-m-d'));
// Call DXCC Worked function to check if the DXCC has been worked before
if (isset($dxccInfo['entity'])) {
$dxccWorkedData = $this->dxccWorked($dxccInfo['entity']);
$record = array_merge($record, $dxccWorkedData);
} else {
// Handle the case where 'entity' is not set in $dxccInfo
$itemsToAdd = array(
'workedBefore' => false,
'confirmed' => false,
);
$record = array_merge($record, $itemsToAdd);
}
$thisWeekRecords[] = $record;
if (($startDate >= $startOfWeek && $startDate <= $endOfWeek) ||
($endDate >= $startOfWeek && $endDate <= $endOfWeek)) {
$weekRecords[] = $record;
}
}
return $thisWeekRecords;
if (empty($weekRecords)) {
return array();
}
// Batch process DXCC lookups
$callsigns = array_column($weekRecords, 'callsign');
$dates = array_column($weekRecords, '0');
$dxccEntities = $this->batchDxccLookup($callsigns, $dates);
// Get worked/confirmed status for all entities in batch
$uniqueEntities = array_unique(array_filter($dxccEntities));
$dxccStatus = $this->batchDxccWorkedStatus($uniqueEntities);
// Process results
foreach ($weekRecords as $index => $record) {
$endDate = new DateTime($record['1']);
$now = new DateTime();
$interval = $now->diff($endDate);
$daysLeft = $interval->days;
$daysLeft = ($daysLeft == 0) ? "Last day" : $daysLeft . " days left";
$record['daysLeft'] = $daysLeft;
$oldStartDate = DateTime::createFromFormat('Y-m-d', $record['0']);
$record['startDate'] = $oldStartDate->format($custom_date_format);
$oldEndDate = DateTime::createFromFormat('Y-m-d', $record['1']);
$record['endDate'] = $oldEndDate->format($custom_date_format);
// Get DXCC status for this callsign
$entity = $dxccEntities[$index] ?? null;
$worked = $entity && isset($dxccStatus[$entity]) ? $dxccStatus[$entity] : ['workedBefore' => false, 'confirmed' => false];
$record['workedBefore'] = $worked['workedBefore'];
$record['confirmed'] = $worked['confirmed'];
$thisWeekRecords[] = $record;
}
return $thisWeekRecords;
}
function dxccWorked($country)