#include <datatable.hpp>

Public Member Functions
	DataTable ()

DataTableStr	csv_read (std::string filename, std::vector< std::string > columns={})
	Reads a CSV file and returns a DataTableStr parses CSV files and stores all data as strings. More...

void	csv_write ()
	Write DataTable to a CSV file. More...

DataTableStr	tsv_read (std::string filename, std::vector< std::string > columns={})
	Reads a TSV file and returns a DataTableStr parses TSV files and stores all data as strings. More...

DataTableStr	json_read (std::string filename, std::vector< std::string > objs={})
	Reads a JSON file and returns a DataTableStr parses JSON files and stores all data as strings. More...

DataTableStr	datetime (std::string column_name, bool extract_year=true, bool extract_month=true, bool extract_time=false)
	Extracts date and time components from a timestamp column. More...

void	sort (const std::vector< std::string > &sort_columns, bool ascending=true)
	Sorts the rows of the DataTable based on specified columns. More...

std::vector< DataTableStr >	group_by (std::vector< std::string > group_by_columns)
	Groups the data by specified columns. More...

DataTableStr	first (const std::vector< gpmp::core::DataTableStr > &groups) const
	Gets the first element of each created group. More...

void	describe ()
	Prints some information about the DataTable. More...

DataTableInt	str_to_int (DataTableStr src)
	Converts a DataTableStr to a DataTableInt. More...

DataTableDouble	str_to_double (DataTableStr src)
	Converts a DataTableStr to a DataTableDouble. More...

template<typename T >
void	display (std::pair< std::vector< T >, std::vector< std::vector< T >>> data, bool display_all=false)
	Sort a DataTable based on a specified column. More...

void	display (bool display_all=false)
	Overload function for display() defaults to displaying what is currently stored in a DataTable object. More...

	DataTable ()
	DataTable constructor. Initializes column & row storage. More...

void	printData ()

TableType	csv_read_new (std::string filename, std::vector< std::string > columns={})
	Reads a CSV file and returns a DataTableStr parses CSV files and stores all data as strings. More...

DataTableStr	csv_read (std::string filename, std::vector< std::string > columns)

void	csv_write ()
	Write DataTable to a CSV file. More...

DataTableStr	tsv_read (std::string filename, std::vector< std::string > columns={})
	Reads a TSV file and returns a DataTableStr parses TSV files and stores all data as strings. More...

DataTableStr	json_read (std::string filename, std::vector< std::string > objs={})
	Reads a JSON file and returns a DataTableStr parses JSON files and stores all data as strings. More...

void	drop (std::vector< std::string > column_name)
	Drop specified rows from a DataTable. More...

DataTableStr	datetime (std::string column_name, bool extract_year=true, bool extract_month=true, bool extract_time=false)
	Extracts date and time components from a timestamp column. More...

void	sort (const std::vector< std::string > &sort_columns, bool ascending=true)
	Sorts the rows of the DataTable based on specified columns. More...

std::vector< DataTableStr >	group_by (std::vector< std::string > group_by_columns)
	Groups the data by specified columns. More...

DataTableStr	first (const std::vector< gpmp::core::DataTableStr > &groups) const
	Gets the first element of each created group. More...

void	describe ()
	Displays some information about the DataTable. More...

void	info ()
	Displays data types and null vals for each column. More...

TableType	native_type (const std::vector< std::string > &skip_columns={})
	Converts DataTable column's rows to their native types. Since the existing DataTable read/load related methods hone in on the DataTableStr type, there must be a way to get those types to their native formats. More...

DataType	inferType (const std::vector< std::string > &column)

DataTableInt	str_to_int (DataTableStr src)
	Converts a DataTableStr to a DataTableInt. More...

DataTableDouble	str_to_double (DataTableStr src)
	Converts a DataTableStr to a DataTableDouble. More...

void	display (const TableType &data, bool display_all=false)
	Sort a DataTable based on a specified column. More...

void	display (bool display_all=false)
	Overload function for display() defaults to displaying what is currently stored in a DataTable object. More...

Private Attributes
std::vector< std::string >	headers_

std::vector< std::vector< std::string > >	rows_

std::vector< std::string >	new_headers_

std::vector< std::vector< std::string > >	data_

DataTableStr	original_data_

MixedType	rows_

MixedType	data_

Detailed Description

Examples: linreg.cpp.

Definition at line 76 of file datatable.hpp.

Constructor & Destructor Documentation

◆ DataTable() [1/2]

gpmp::core::DataTable::DataTable ( )

inline

Definition at line 91 of file datatable.hpp.

                 {
         // Initialize data_ and headers_ to empty vectors
         data_ = std::vector<std::vector<std::string>>();
         headers_ = std::vector<std::string>();
     }

References data_, and headers_.

◆ DataTable() [2/2]

gpmp::core::DataTable::DataTable ( )

inline

DataTable constructor. Initializes column & row storage.

Definition at line 111 of file datatable_wip.hpp.

                 {
         // Initialize data_ and headers_ to empty vectors
         headers_ = std::vector<std::string>();
         data_ = MixedType();
     }

References data_, and headers_.

Member Function Documentation

◆ csv_read() [1/2]

DataTableStr gpmp::core::DataTable::csv_read	(	std::string	filename,
		std::vector< std::string >	columns
	)

◆ csv_read() [2/2]

gpmp::core::TableType gpmp::core::DataTable::csv_read	(	std::string	filename,
		std::vector< std::string >	columns = `{}`
	)

Reads a CSV file and returns a DataTableStr parses CSV files and stores all data as strings.

Parameters

filename	the path to the CSV file
columns	optional vector of column names to read in, if empty all columns will be read in

Returns: a DataTableStr containing the column names and data

Examples: linreg.cpp.

Definition at line 57 of file datatable.cpp.

                                                               {
  
     int fd = open(filename.c_str(), O_RDONLY);
     if (fd == -1) {
         // Handle file open error
         perror("Error opening file");
         exit(EXIT_FAILURE);
     }
  
     off_t size = lseek(fd, 0, SEEK_END);
     char *file_data =
         static_cast<char *>(mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0));
  
     if (file_data == MAP_FAILED) {
         // Handle memory mapping error
         perror("Error mapping file to memory");
         close(fd);
         exit(EXIT_FAILURE);
     }
  
     std::stringstream file_stream(file_data);
     std::vector<std::vector<std::string>> data;
     std::string line;
  
     // Get the header line and parse the column names
     getline(file_stream, line);
     std::stringstream header(line);
     std::vector<std::string> header_cols;
     std::string columnName;
  
     while (getline(header, columnName, ',')) {
         header_cols.push_back(columnName);
     }
  
     // If no columns are specified, read in all columns
     if (columns.empty()) {
         columns = header_cols;
     }
  
     // Check if specified columns exist in the header
     for (const auto &column : columns) {
         if (std::find(header_cols.begin(), header_cols.end(), column) ==
             header_cols.end()) {
             // Handle column not found error
             perror(("Column: " + column + " not found").c_str());
             munmap(file_data, size);
             close(fd);
             exit(EXIT_FAILURE);
         }
     }
  
     // Read in the data rows
     while (getline(file_stream, line)) {
         std::vector<std::string> row;
         std::stringstream rowStream(line);
         std::string value;
         int columnIndex = 0;
  
         while (getline(rowStream, value, ',')) {
             // If column is specified, only read in specified columns
             if (std::find(columns.begin(),
                           columns.end(),
                           header_cols[columnIndex]) != columns.end()) {
                 row.push_back(value);
             }
  
             columnIndex++;
         }
  
         if (!row.empty()) {
             data.push_back(row);
         }
     }
  
     // populate headers_ class variable
     headers_ = columns;
     // populate data_ class variable
     data_ = data;
  
     munmap(file_data, size);
     close(fd);
  
     return std::make_pair(headers_, data_);
 }

References data_, and headers_.

Referenced by main(), and test_train().

◆ csv_read_new()

TableType gpmp::core::DataTable::csv_read_new	(	std::string	filename,
		std::vector< std::string >	columns = `{}`
	)

Reads a CSV file and returns a DataTableStr parses CSV files and stores all data as strings.

Parameters

filename	the path to the CSV file
columns	optional vector of column names to read in, if empty all columns will be read in

Returns: a DataTableStr containing the column names and data

◆ csv_write() [1/2]

void gpmp::core::DataTable::csv_write ( )

Write DataTable to a CSV file.

◆ csv_write() [2/2]

void gpmp::core::DataTable::csv_write ( )

Write DataTable to a CSV file.

◆ datetime() [1/2]

gpmp::core::DataTableStr gpmp::core::DataTable::datetime	(	std::string	column_name,
		bool	extract_year = `true`,
		bool	extract_month = `true`,
		bool	extract_time = `false`
	)

Extracts date and time components from a timestamp column.

Parameters

column_name	The name of the timestamp column
extract_year	If true, extract the year component
extract_month	If true, extract the month component
extract_time	If true, extract the time component

Returns: A new DataTableStr with extracted components

Definition at line 147 of file datatable.cpp.

                                                    {
     // Find the index of the specified column
     auto column_iter = std::find(headers_.begin(), headers_.end(), column_name);
     if (column_iter == headers_.end()) {
         _log_.log(ERROR, "Column: " + column_name + " node found");
         exit(EXIT_FAILURE);
     }
     int column_index = std::distance(headers_.begin(), column_iter);
  
     // Extract components from each row
     std::vector<std::string> new_headers = headers_;
     std::vector<std::vector<std::string>> new_data;
  
     // Iterate and populate the additional columns
     for (size_t row_index = 0; row_index < data_.size(); ++row_index) {
         std::vector<std::string> row = data_[row_index];
         // If column row is not found
         if (row.size() <= static_cast<size_t>(column_index)) {
             _log_.log(ERROR, "Column: " + column_name + " not found");
  
             exit(EXIT_FAILURE);
         }
  
         std::string timestamp = row[column_index];
         std::string year, month, time;
  
         // Create a new row with extracted components
         std::vector<std::string> new_row;
  
         // Extract year, month, and time components
         if (extract_year) {
             year = timestamp.substr(timestamp.find_last_of('/') + 1, 4);
             new_row.push_back(year);
         }
         if (extract_month) {
             month = timestamp.substr(0, timestamp.find_first_of('/'));
             new_row.push_back(month);
         }
         if (extract_time) {
             time = timestamp.substr(timestamp.find(' ') + 1);
             new_row.push_back(time);
         }
  
         // append original row data
         new_row.insert(new_row.end(), row.begin(), row.end());
         // add new rows
         new_data.push_back(new_row);
     }
  
     // Create new headers based on the extracted components
     if (extract_month)
         new_headers.insert(new_headers.begin(), "Month");
     if (extract_year)
         new_headers.insert(new_headers.begin(), "Year");
     if (extract_time)
         new_headers.insert(new_headers.begin(), "Time");
  
     // set class car data_ to hold rows/lines
     data_ = new_data;
     // set class var modified headers to new headers
     // new_headers_ = new_headers;
     headers_ = new_headers;
  
     return std::make_pair(new_headers, new_data);
 }

References _log_, ERROR, gpmp::core::Logger::log(), and year.

◆ datetime() [2/2]

DataTableStr gpmp::core::DataTable::datetime	(	std::string	column_name,
		bool	extract_year = `true`,
		bool	extract_month = `true`,
		bool	extract_time = `false`
	)

Extracts date and time components from a timestamp column.

Parameters

column_name	The name of the timestamp column
extract_year	If true, extract the year component
extract_month	If true, extract the month component
extract_time	If true, extract the time component

Returns: A new DataTableStr with extracted components

◆ describe() [1/2]

void gpmp::core::DataTable::describe ( )

Prints some information about the DataTable.

◆ describe() [2/2]

void gpmp::core::DataTable::describe ( )

Displays some information about the DataTable.

◆ display() [1/4]

void gpmp::core::DataTable::display ( bool display_all = false )

inline

Overload function for display() defaults to displaying what is currently stored in a DataTable object.

Parameters

display_all Display all rows, defaults to false.

Definition at line 316 of file datatable.hpp.

                                            {
         display(std::make_pair(headers_, data_), display_all);
     }

References data_, display(), and headers_.

◆ display() [2/4]

void gpmp::core::DataTable::display ( bool display_all = false )

Overload function for display() defaults to displaying what is currently stored in a DataTable object.

Parameters

display_all Display all rows, defaults to false.

◆ display() [3/4]

void gpmp::core::DataTable::display	(	const TableType &	data,
		bool	display_all = `false`
	)

Sort a DataTable based on a specified column.

Displays a DataTable of type T with the option to display all or a subset of rows

Template Parameters

T	The type of the DataTable to be displayed

Parameters

data	A pair of vectors representing the header and data rows of the DataTable
display_all	A flag indicating whether to display all rows or just a subset

Definition at line 196 of file datatable2.cpp.

                                                       {
     int num_columns = data.first.size();
     int num_rows = data.second.size();
     int num_omitted_rows = 0;
  
     std::vector<int> max_column_widths(num_columns, 0);
  
     // Calculate the maximum width for each column based on column headers
     for (int i = 0; i < num_columns; i++) {
         max_column_widths[i] = data.first[i].length();
     }
  
     // Calculate the maximum width for each column based on data rows
     for (int i = 0; i < num_columns; i++) {
         for (const auto &row : data.second) {
             if (i < static_cast<int>(row.size())) {
                 std::visit(
                     [&max_column_widths, &i](const auto &cellValue) {
                         using T = std::decay_t<decltype(cellValue)>;
                         if constexpr (std::is_same_v<T, std::string>) {
                             max_column_widths[i] =
                                 std::max(max_column_widths[i],
                                          static_cast<int>(cellValue.length()));
                         } else if constexpr (std::is_integral_v<T> ||
                                              std::is_floating_point_v<T>) {
                             max_column_widths[i] = std::max(
                                 max_column_widths[i],
                                 static_cast<int>(
                                     std::to_string(cellValue).length()));
                         }
                     },
                     row[i]);
             }
         }
     }
  
     const int dateTimeColumnIndex = 0;
     max_column_widths[dateTimeColumnIndex] =
         std::max(max_column_widths[dateTimeColumnIndex], 0);
  
     // Define a function to print a row
     auto printRow = [&data, &max_column_widths, num_columns](int row_index) {
         std::cout << std::setw(7) << std::right << row_index << "  ";
  
         for (int j = 0; j < num_columns; j++) {
             if (j < static_cast<int>(data.second[row_index].size())) {
                 std::visit(
                     [&max_column_widths, &j](const auto &cellValue) {
                         using T = std::decay_t<decltype(cellValue)>;
                         if constexpr (std::is_same_v<T, double> ||
                                       std::is_same_v<T, long double>) {
                             // Convert the value to a string without trailing
                             // zeros
                             std::string cellValueStr =
                                 std::to_string(cellValue);
                             cellValueStr.erase(
                                 cellValueStr.find_last_not_of('0') + 1,
                                 std::string::npos);
                             cellValueStr.erase(
                                 cellValueStr.find_last_not_of('.') + 1,
                                 std::string::npos);
  
                             std::cout << std::setw(max_column_widths[j])
                                       << std::right << cellValueStr << "  ";
                         } else {
                             std::cout << std::setw(max_column_widths[j])
                                       << std::right << cellValue << "  ";
                         }
                     },
                     data.second[row_index][j]);
             }
         }
  
         std::cout << std::endl;
     };
  
     // Print headers
     std::cout << std::setw(7) << std::right << "Index"
               << "  ";
     for (int i = 0; i < num_columns; i++) {
         std::cout << std::setw(max_column_widths[i]) << std::right
                   << data.first[i] << "  ";
     }
     std::cout << std::endl;
  
     int num_elements = data.second.size();
     if (!display_all && num_elements > MAX_ROWS) {
         for (int i = 0; i < SHOW_ROWS; i++) {
             printRow(i);
         }
         num_omitted_rows = num_elements - MAX_ROWS;
         std::cout << "...\n";
         std::cout << "[" << num_omitted_rows << " rows omitted]\n";
         for (int i = num_elements - SHOW_ROWS; i < num_elements; i++) {
             printRow(i);
         }
     } else {
         // Print all rows
         for (int i = 0; i < num_elements; i++) {
             printRow(i);
         }
     }
  
     // Print the number of rows and columns
     std::cout << "[" << num_rows << " rows"
               << " x " << num_columns << " columns";
     std::cout << "]\n\n";
 }

References MAX_ROWS, and SHOW_ROWS.

◆ display() [4/4]

template<typename T >

void gpmp::core::DataTable::display	(	std::pair< std::vector< T >, std::vector< std::vector< T >>>	data,
		bool	display_all = `false`
	)

inline

Sort a DataTable based on a specified column.

Displays a DataTable of type T with the option to display all or a subset of rows

Template Parameters

T	The type of the DataTable to be displayed

Parameters

data	A pair of vectors representing the header and data rows of the DataTable
display_all	A flag indicating whether to display all rows or just a subset

Definition at line 216 of file datatable.hpp.

                                            {
         // Get the number of columns and rows in the data
         int num_columns = data.first.size();
         int num_rows = data.second.size();
         int num_omitted_rows = 0;
  
         // Initialize max_column_widths with the lengths of column headers
         std::vector<int> max_column_widths(num_columns, 0);
  
         // Calculate the maximum width for each column based on column headers
         for (int i = 0; i < num_columns; i++) {
             max_column_widths[i] = data.first[i].length();
         }
  
         // Calculate the maximum width for each column based on data rows
         for (int i = 0; i < num_columns; i++) {
             for (const auto &row : data.second) {
                 if (i < static_cast<int>(row.size())) {
                     max_column_widths[i] =
                         std::max(max_column_widths[i],
                                  static_cast<int>(row[i].length()));
                 }
             }
         }
  
         // Set a larger width for the DateTime column (adjust the index as
         // needed later on)
         const int dateTimeColumnIndex = 0;
         // adjust as needed?
         max_column_widths[dateTimeColumnIndex] =
             std::max(max_column_widths[dateTimeColumnIndex], 0);
  
         // Print headers with right-aligned values
         std::cout << std::setw(7) << std::right << "Index"
                   << "  ";
  
         for (int i = 0; i < num_columns; i++) {
             std::cout << std::setw(max_column_widths[i]) << std::right
                       << data.first[i] << "  ";
         }
         std::cout << std::endl;
  
         int num_elements = data.second.size();
         if (!display_all && num_elements > MAX_ROWS) {
             for (int i = 0; i < SHOW_ROWS; i++) {
                 // Prit index
                 std::cout << std::setw(7) << std::right << i << "  ";
                 // Print each row with right-aligned values
                 for (int j = 0; j < num_columns; j++) {
                     if (j < static_cast<int>(data.second[i].size())) {
                         std::cout << std::setw(max_column_widths[j])
                                   << std::right << data.second[i][j] << "  ";
                     }
                 }
                 std::cout << std::endl;
             }
             num_omitted_rows = num_elements - MAX_ROWS;
             std::cout << "...\n";
             std::cout << "[" << num_omitted_rows << " rows omitted]\n";
             for (int i = num_elements - SHOW_ROWS; i < num_elements; i++) {
                 std::cout << std::setw(7) << std::right << i << "  ";
                 // Print each row with right-aligned values
                 for (int j = 0; j < num_columns; j++) {
                     if (j < static_cast<int>(data.second[i].size())) {
                         std::cout << std::setw(max_column_widths[j])
                                   << std::right << data.second[i][j] << "  ";
                     }
                 }
                 std::cout << std::endl;
             }
         } else {
             // Print all rows with right-aligned values
             for (int i = 0; i < num_elements; i++) {
  
                 // Print index
                 std::cout << std::setw(7) << std::right << i << "  ";
                 for (int j = 0; j < num_columns; j++) {
                     if (j < static_cast<int>(data.second[i].size())) {
  
                         // Print formatted row
                         std::cout << std::setw(max_column_widths[j])
                                   << std::right << data.second[i][j] << "  ";
                     }
                 }
                 std::cout << std::endl;
             }
         }
  
         // Print the number of rows and columns
         std::cout << "[" << num_rows << " rows"
                   << " x " << num_columns << " columns";
         std::cout << "]\n\n";
     }

References MAX_ROWS, and SHOW_ROWS.

Referenced by display(), and main().

◆ drop()

void gpmp::core::DataTable::drop ( std::vector< std::string > column_name )

Drop specified rows from a DataTable.

◆ first() [1/2]

gpmp::core::DataTableStr gpmp::core::DataTable::first ( const std::vector< gpmp::core::DataTableStr > & groups ) const

Gets the first element of each created group.

Parameters

groups Return type of gpmp::core::DataTable.group()

Returns: a DataTableStr

Definition at line 317 of file datatable.cpp.

                                                          {
     if (groups.empty()) {
         // Handle the case when there are no groups
         return std::make_pair(std::vector<std::string>(),
                               std::vector<std::vector<std::string>>());
     }
  
     std::vector<std::vector<std::string>> first_rows;
  
     for (const gpmp::core::DataTableStr &group : groups) {
         if (!group.second.empty()) {
             first_rows.push_back(
                 group.second[0]); // Get the first row of each group
         }
     }
  
     if (!first_rows.empty()) {
         // Assuming all groups have the same headers as the first group
         return std::make_pair(groups[0].first, first_rows);
     } else {
         // Handle the case when there are no first rows found.
         return std::make_pair(groups[0].first,
                               std::vector<std::vector<std::string>>());
     }
 }

◆ first() [2/2]

DataTableStr gpmp::core::DataTable::first ( const std::vector< gpmp::core::DataTableStr > & groups ) const

Gets the first element of each created group.

Parameters

groups Return type of gpmp::core::DataTable.group()

Returns: a DataTableStr

◆ group_by() [1/2]

std::vector< gpmp::core::DataTableStr > gpmp::core::DataTable::group_by ( std::vector< std::string > group_by_columns )

Groups the data by specified columns.

Parameters

group_by_columns The column names to group by

Returns: A vector of DataTableStr, each containing a group of rows

Definition at line 250 of file datatable.cpp.

                                                                  {
     // Find the indices of the specified group by columns
     std::vector<int> group_by_indices;
  
     // Traverse group column names
     for (const std::string &column_name : group_by_columns) {
         std::cout << "Searching for column: " << column_name << std::endl;
  
         // Find start/end and match column name
         auto column_iter =
             std::find(headers_.begin(), headers_.end(), column_name);
  
         // If no columns
         if (column_iter == headers_.end()) {
             _log_.log(ERROR, "Column: " + column_name + " not found");
             exit(EXIT_FAILURE);
         }
         // column index set to distance from start of first col to nexter iter
         int column_index = std::distance(headers_.begin(), column_iter);
         // add column index to group
         group_by_indices.push_back(column_index);
     }
  
     // Group the data based on the specified columns using a vector
     std::vector<std::pair<std::vector<std::string>, gpmp::core::DataTableStr>>
         groups;
  
     // Traverse row/line data
     for (const std::vector<std::string> &row : data_) {
         // store group key for each row
         std::vector<std::string> group_key;
         // Fill group key from specified group column names
         for (int index : group_by_indices) {
             group_key.push_back(row[index]);
         }
  
         // Check if the group already exists
         auto group_iter = std::find_if(
             groups.begin(),
             groups.end(),
             [&group_key](const std::pair<std::vector<std::string>,
                                          gpmp::core::DataTableStr> &group) {
                 return group.first == group_key;
             });
         // If the group DNE create a new one to add to groups vector
         if (group_iter == groups.end()) {
             // Create a new group
             groups.push_back(
                 {group_key, gpmp::core::DataTableStr(headers_, {})});
             group_iter = groups.end() - 1;
         }
         // Add current row to group
         group_iter->second.second.push_back(row);
     }
  
     // Extract the grouped data into a vector
     std::vector<gpmp::core::DataTableStr> grouped_data;
     // Iterate over sorted groups to push onto result vector
     for (const auto &group : groups) {
         grouped_data.push_back(group.second);
     }
  
     // Return final DataTableStr type
     return grouped_data;
 }

References _log_, ERROR, and gpmp::core::Logger::log().

◆ group_by() [2/2]

std::vector<DataTableStr> gpmp::core::DataTable::group_by ( std::vector< std::string > group_by_columns )

Groups the data by specified columns.

Parameters

group_by_columns The column names to group by

Returns: A vector of DataTableStr, each containing a group of rows

◆ inferType()

gpmp::core::DataType gpmp::core::DataTable::inferType ( const std::vector< std::string > & column )

Definition at line 135 of file datatable1.cpp.

                                                                {
     int integer_count = 0;
     int double_count = 0;
     int string_count = 0;
  
     for (const std::string &cell : column) {
         if (is_int(cell)) {
             integer_count++;
         } else if (is_double(cell)) {
             double_count++;
         } else {
             string_count++;
         }
     }
  
     _log_.log(INFO,
               "int/double/str: " + std::to_string(integer_count) + "/" +
                   std::to_string(double_count) + "/" +
                   std::to_string(string_count));
  
     if (integer_count > double_count) {
         return DataType::dt_int32;
     } else if (double_count > integer_count) {
         return DataType::dt_double;
     } else {
         return DataType::dt_str;
     }
 }

References _log_, gpmp::core::dt_double, gpmp::core::dt_int32, gpmp::core::dt_str, INFO, is_double(), is_int(), and gpmp::core::Logger::log().

◆ info()

void gpmp::core::DataTable::info ( )

Displays data types and null vals for each column.

Definition at line 647 of file datatable2.cpp.

                              {
     // Calculate memory usage for each column and keep track of data type
     std::vector<double> column_memory_usages(headers_.size(), 0.0);
     std::vector<std::string> column_data_types(headers_.size());
     double total_memory_usage_kb = 0.0;
  
     // Calculate memory usage in bytes for the entire table
     size_t memory_usage_bytes = sizeof(headers_);
     for (const auto &row : data_) {
         for (size_t i = 0; i < row.size(); ++i) {
             if (std::holds_alternative<int64_t>(row[i])) {
                 memory_usage_bytes += sizeof(int64_t);
                 column_memory_usages[i] +=
                     static_cast<double>(sizeof(int64_t)) / 1024.0;
                 column_data_types[i] = "int64_t";
             } else if (std::holds_alternative<long double>(row[i])) {
                 memory_usage_bytes += sizeof(long double);
                 column_memory_usages[i] +=
                     static_cast<double>(sizeof(long double)) / 1024.0;
                 column_data_types[i] = "long double";
             } else if (std::holds_alternative<std::string>(row[i])) {
                 memory_usage_bytes += std::get<std::string>(row[i]).capacity();
                 column_memory_usages[i] +=
                     static_cast<double>(
                         std::get<std::string>(row[i]).capacity()) /
                     1024.0;
                 column_data_types[i] = "std::string";
             }
         }
     }
  
     // Convert total memory usage to KB
     total_memory_usage_kb = static_cast<double>(memory_usage_bytes) / 1024.0;
  
     // Find the maximum column name length
     size_t max_column_name_length = 0;
     for (const std::string &column : headers_) {
         max_column_name_length =
             std::max(max_column_name_length, column.length());
     }
  
     // Find the maximum data type length
     size_t max_data_type_length = 0;
     for (const std::string &data_type : column_data_types) {
         max_data_type_length =
             std::max(max_data_type_length, data_type.length());
     }
  
     // Set the column width for formatting
     int column_width = static_cast<int>(std::max(max_column_name_length,
                                                  max_data_type_length)) +
                        2; // Add extra padding
  
     // Print header
     std::cout << std::left << std::setw(column_width) << "Column"
               << std::setw(column_width) << "Type" << std::setw(column_width)
               << "Memory Usage (KB)" << std::endl;
  
     // Print data
     for (size_t i = 0; i < headers_.size(); ++i) {
         std::cout << std::left << std::setw(column_width) << headers_[i]
                   << std::setw(column_width) << column_data_types[i]
                   << std::setw(column_width) << std::fixed
                   << std::setprecision(2) << column_memory_usages[i]
                   << std::endl;
     }
  
     // Print total table memory usage
     std::cout << "\nTotal Memory Usage: " << std::fixed << std::setprecision(2)
               << total_memory_usage_kb << " KB" << std::endl;
 }

◆ json_read() [1/2]

DataTableStr gpmp::core::DataTable::json_read	(	std::string	filename,
		std::vector< std::string >	objs = `{}`
	)

Reads a JSON file and returns a DataTableStr parses JSON files and stores all data as strings.

Parameters

filename	the path to the JSON file
objs	optional vector of JSON object names to read in, if empty all objects will be read in

Returns: a DataTableStr containing the column names and data

◆ json_read() [2/2]

DataTableStr gpmp::core::DataTable::json_read	(	std::string	filename,
		std::vector< std::string >	objs = `{}`
	)

Reads a JSON file and returns a DataTableStr parses JSON files and stores all data as strings.

Parameters

filename	the path to the JSON file
objs	optional vector of JSON object names to read in, if empty all objects will be read in

Returns: a DataTableStr containing the column names and data

◆ native_type()

gpmp::core::TableType gpmp::core::DataTable::native_type ( const std::vector< std::string > & skip_columns = {} )

Converts DataTable column's rows to their native types. Since the existing DataTable read/load related methods hone in on the DataTableStr type, there must be a way to get those types to their native formats.

Definition at line 176 of file datatable1.cpp.

                                               {
     gpmp::core::TableType mixed_data;
  
     // Include all column headers in mixed_data (including skipped ones)
     mixed_data.first = headers_;
  
     std::vector<gpmp::core::DataType> column_data_types;
  
     // Determine data types for each column (skip_columns remain as strings)
     for (size_t col = 0; col < headers_.size(); ++col) {
         // Check if this column should be skipped
         if (std::find(skip_columns.begin(),
                       skip_columns.end(),
                       headers_[col]) != skip_columns.end()) {
             column_data_types.push_back(gpmp::core::DataType::dt_str);
             _log_.log(INFO, "Skipping column: " + headers_[col]);
         } else {
             std::vector<std::string> column_data;
             for (const std::vector<std::string> &rowData : data_) {
                 column_data.push_back(rowData[col]);
             }
             gpmp::core::DataType column_type = inferType(column_data);
             column_data_types.push_back(column_type);
  
             _log_.log(INFO,
                       "Column " + headers_[col] +
                           " using type: " + dt_to_str(column_type));
         }
     }
  
     // Traverse rows and convert based on the determined data types
     for (const std::vector<std::string> &row : data_) {
         std::vector<std::variant<int64_t, long double, std::string>> mixed_row;
  
         for (size_t col = 0; col < headers_.size(); ++col) {
             const std::string &cell = row[col];
             gpmp::core::DataType column_type = column_data_types[col];
  
             if (column_type == gpmp::core::DataType::dt_int32) {
  
                 mixed_row.push_back(std::stoi(cell));
             } else if (column_type == gpmp::core::DataType::dt_double) {
                 mixed_row.push_back(std::stold(cell));
             } else {
                 mixed_row.push_back(cell); // Keep as a string
             }
         }
  
         mixed_data.second.push_back(mixed_row);
     }
  
     std::cout << "Mixed Data:" << std::endl;
     for (const std::string &header : mixed_data.first) {
         std::cout << header << " ";
     }
     std::cout << std::endl;
  
     for (const auto &row : mixed_data.second) {
         for (const auto &cell : row) {
             if (std::holds_alternative<int64_t>(cell)) {
                 std::cout << std::get<int64_t>(cell) << " ";
             } else if (std::holds_alternative<long double>(cell)) {
                 std::cout << std::get<long double>(cell) << " ";
             } else if (std::holds_alternative<std::string>(cell)) {
                 std::cout << std::get<std::string>(cell) << " ";
             }
         }
         std::cout << std::endl;
     }
  
     return mixed_data;
 }

References _log_, gpmp::core::dt_double, gpmp::core::dt_int32, gpmp::core::dt_str, dt_to_str(), INFO, and gpmp::core::Logger::log().

◆ printData()

void gpmp::core::DataTable::printData ( )

inline

Definition at line 116 of file datatable_wip.hpp.

                      {
         // Print column headers
         for (const auto &header : headers_) {
             std::cout << header << "\t";
         }
         std::cout << std::endl;
  
         // Print data rows
         for (const auto &row : data_) {
             for (const auto &cell : row) {
                 // Check the type of cell and print accordingly
                 if (std::holds_alternative<int64_t>(cell)) {
                     std::cout << std::get<int64_t>(cell);
                 } else if (std::holds_alternative<long double>(cell)) {
                     std::cout << std::get<long double>(cell);
                 } else if (std::holds_alternative<std::string>(cell)) {
                     std::cout << std::get<std::string>(cell);
                 }
  
                 std::cout << "\t";
             }
             std::cout << std::endl;
         }
     }

References data_, and headers_.

◆ sort() [1/2]

void gpmp::core::DataTable::sort	(	const std::vector< std::string > &	sort_columns,
		bool	ascending = `true`
	)

Sorts the rows of the DataTable based on specified columns.

Parameters

sort_columns	A vector of column names to sort by.
ascending	If true, sort in ascending order; otherwise, sort in descending order, default is true.

Definition at line 217 of file datatable.cpp.

                                                  {
     // Extract the column indices to be sorted by from the original data
     std::vector<size_t> column_indices;
     for (const std::string &column : sort_columns) {
         auto iter = std::find(headers_.begin(), headers_.end(), column);
         if (iter != headers_.end()) {
             size_t index = std::distance(headers_.begin(), iter);
             column_indices.push_back(index);
         }
     }
  
     // Sort the data based on the specified columns
     std::stable_sort(data_.begin(),
                      data_.end(),
                      [&](const std::vector<std::string> &row1,
                          const std::vector<std::string> &row2) {
                          for (size_t index : column_indices) {
                              if (row1[index] != row2[index]) {
                                  if (ascending) {
                                      return row1[index] < row2[index];
                                  } else {
                                      return row1[index] > row2[index];
                                  }
                              }
                          }
                          // Rows are equal, nothing to sort
                          return false;
                      });
 }

◆ sort() [2/2]

void gpmp::core::DataTable::sort	(	const std::vector< std::string > &	sort_columns,
		bool	ascending = `true`
	)

Sorts the rows of the DataTable based on specified columns.

Parameters

sort_columns	A vector of column names to sort by.
ascending	If true, sort in ascending order; otherwise, sort in descending order, default is true.

◆ str_to_double() [1/2]

gpmp::core::DataTableDouble gpmp::core::DataTable::str_to_double ( DataTableStr src )

Converts a DataTableStr to a DataTableDouble.

Parameters

src	A DataTableStr object to be converted

Returns: The converted DataTableDouble

Note: The function assumes the input DataTableStr contains only valid double type elements

Definition at line 370 of file datatable.cpp.

                                                          {
     gpmp::core::DataTableDouble dest;
  
     for (const auto &v : src.first) {
         if (std::regex_match(v, std::regex("[-+]?\\d*\\.?\\d+"))) {
             dest.first.push_back(std::stold(v));
         }
     }
  
     for (const auto &vv : src.second) {
         std::vector<long double> new_vec;
         for (const auto &v : vv) {
             if (std::regex_match(v, std::regex("[-+]?\\d*\\.?\\d+"))) {
                 new_vec.push_back(std::stold(v));
             }
         }
         dest.second.push_back(new_vec);
     }
  
     return dest;
 }

◆ str_to_double() [2/2]

DataTableDouble gpmp::core::DataTable::str_to_double ( DataTableStr src )

Converts a DataTableStr to a DataTableDouble.

Parameters

src	A DataTableStr object to be converted

Returns: The converted DataTableDouble

Note: The function assumes the input DataTableStr contains only valid double type elements

◆ str_to_int() [1/2]

gpmp::core::DataTableInt gpmp::core::DataTable::str_to_int ( DataTableStr src )

Converts a DataTableStr to a DataTableInt.

Parameters

src	The DataTableStr to convert

Returns: The converted DataTableInt

Note: This function assumes that the input DataTableStr contains only elements that can be converted to a 64-bit integer using std::stoi() @TODO allow for specific columns to be converted @TODO make use of ThreadPool

Definition at line 347 of file datatable.cpp.

                                                       {
     gpmp::core::DataTableInt dest;
  
     for (const auto &v : src.first) {
         // check if v contains only digits
         if (std::regex_match(v, std::regex("\\d+"))) {
             dest.first.push_back(std::stoi(v));
         }
     }
     for (const auto &vv : src.second) {
         std::vector<int64_t> new_vec;
         for (const auto &v : vv) {
             // check if v contains only digits
             if (std::regex_match(v, std::regex("\\d+"))) {
                 new_vec.push_back(std::stoi(v));
             }
         }
         dest.second.push_back(new_vec);
     }
     return dest;
 }

◆ str_to_int() [2/2]

DataTableInt gpmp::core::DataTable::str_to_int ( DataTableStr src )

Converts a DataTableStr to a DataTableInt.

Parameters

src	The DataTableStr to convert

Returns: The converted DataTableInt

Note: This function assumes that the input DataTableStr contains only elements that can be converted to a 64-bit integer using std::stoi() @TODO allow for specific columns to be converted @TODO make use of ThreadPool

◆ tsv_read() [1/2]

DataTableStr gpmp::core::DataTable::tsv_read	(	std::string	filename,
		std::vector< std::string >	columns = `{}`
	)

Reads a TSV file and returns a DataTableStr parses TSV files and stores all data as strings.

Parameters

filename	the path to the TSV file
columns	optional vector of column names to read in, if empty all columns will be read in

Returns: a DataTableStr containing the column names and data

◆ tsv_read() [2/2]

DataTableStr gpmp::core::DataTable::tsv_read	(	std::string	filename,
		std::vector< std::string >	columns = `{}`
	)

Reads a TSV file and returns a DataTableStr parses TSV files and stores all data as strings.

Parameters

filename	the path to the TSV file
columns	optional vector of column names to read in, if empty all columns will be read in

Returns: a DataTableStr containing the column names and data

Member Data Documentation

◆ data_ [1/2]

std::vector<std::vector<std::string> > gpmp::core::DataTable::data_

private

Definition at line 85 of file datatable.hpp.

Referenced by csv_read(), DataTable(), display(), and printData().

◆ data_ [2/2]

MixedType gpmp::core::DataTable::data_

private

Definition at line 102 of file datatable_wip.hpp.

◆ headers_

std::vector< std::string > gpmp::core::DataTable::headers_

private

Definition at line 79 of file datatable.hpp.

Referenced by csv_read(), DataTable(), display(), and printData().

◆ new_headers_

std::vector< std::string > gpmp::core::DataTable::new_headers_

private

Definition at line 83 of file datatable.hpp.

◆ original_data_

DataTableStr gpmp::core::DataTable::original_data_

private

Definition at line 88 of file datatable.hpp.

◆ rows_ [1/2]

std::vector<std::vector<std::string> > gpmp::core::DataTable::rows_

private

Definition at line 81 of file datatable.hpp.

◆ rows_ [2/2]

MixedType gpmp::core::DataTable::rows_

private

Definition at line 98 of file datatable_wip.hpp.

The documentation for this class was generated from the following files:

include/openGPMP/core/datatable.hpp
include/openGPMP/core/datatable_wip.hpp
modules/core/datatable.cpp
modules/core/datatable1.cpp
modules/core/datatable2.cpp

Public Member Functions

Private Attributes

Detailed Description

Constructor & Destructor Documentation

◆ DataTable() [1/2]

◆ DataTable() [2/2]

Member Function Documentation

◆ csv_read() [1/2]

◆ csv_read() [2/2]

◆ csv_read_new()

◆ csv_write() [1/2]

◆ csv_write() [2/2]

◆ datetime() [1/2]

◆ datetime() [2/2]

◆ describe() [1/2]

◆ describe() [2/2]

◆ display() [1/4]

◆ display() [2/4]

◆ display() [3/4]

◆ display() [4/4]

◆ drop()

◆ first() [1/2]

◆ first() [2/2]

◆ group_by() [1/2]

◆ group_by() [2/2]

◆ inferType()

◆ info()

◆ json_read() [1/2]

◆ json_read() [2/2]

◆ native_type()

◆ printData()

◆ sort() [1/2]

◆ sort() [2/2]

◆ str_to_double() [1/2]

◆ str_to_double() [2/2]

◆ str_to_int() [1/2]

◆ str_to_int() [2/2]

◆ tsv_read() [1/2]

◆ tsv_read() [2/2]

Member Data Documentation

◆ data_ [1/2]

◆ data_ [2/2]

◆ headers_

◆ new_headers_

◆ original_data_

◆ rows_ [1/2]

◆ rows_ [2/2]