fml  0.1-0
Fused Matrix Library
io.hh
1 // This file is part of fml which is released under the Boost Software
2 // License, Version 1.0. See accompanying file LICENSE or copy at
3 // https://www.boost.org/LICENSE_1_0.txt
4 
5 #ifndef FML_CPU_CUTURE_IO_H
6 #define FML_CPU_CUTURE_IO_H
7 #pragma once
8 
9 
10 #include <cstdio>
11 #include <stdexcept>
12 
13 #include "../cpumat.hh"
14 
15 
16 namespace fml
17 {
18 namespace io
19 {
20  namespace
21  {
22  static const int HEADER_LEN = 10;
23  static const char *header = "FMLCPUMAT";
24  }
25 
26 
27 
28  template <typename REAL>
29  void write_cpu(const char *filename, const cpumat<REAL> &x)
30  {
31  FILE *f;
32  f = fopen(filename, "wb");
33  if (!f)
34  throw std::runtime_error("unable to open file for writing");
35 
36  len_t m = x.nrows();
37  len_t n = x.ncols();
38  size_t len = (size_t) m*n;
39 
40  fwrite(header, sizeof(char), HEADER_LEN, f);
41  fwrite(&m, sizeof(len_t), 1, f);
42  fwrite(&n, sizeof(len_t), 1, f);
43  fwrite(x.data_ptr(), sizeof(REAL), len, f);
44  fclose(f);
45  }
46 
47 
48 
49  static inline long int read_offset()
50  {
51  return (long int) HEADER_LEN + 2*sizeof(len_t);
52  }
53 
54 
55 
56  template <typename REAL>
57  void read_cpu(const char *filename, cpumat<REAL> &x)
58  {
59  FILE *f;
60  f = fopen(filename, "rb");
61  if (!f)
62  throw std::runtime_error("unable to open file for reading");
63 
64  char tmp[HEADER_LEN];
65  int nread = fread(tmp, sizeof(char), HEADER_LEN, f);
66  if (nread != HEADER_LEN || strcmp(tmp, header))
67  throw std::runtime_error("mal-formed file");
68 
69  len_t m, n;
70  nread = fread(&m, sizeof(len_t), 1, f);
71  nread = fread(&n, sizeof(len_t), 1, f);
72 
73  x.resize(m, n);
74  size_t len = (size_t) m*n;
75 
76  nread = fread(x.data_ptr(), sizeof(REAL), len, f);
77  fclose(f);
78  }
79 
80 
81  template <typename REAL>
82  void read_cpu_chunk(const char *filename, const len_t row_first, const len_t row_last, cpumat<REAL> &x)
83  {
84  FILE *f;
85  f = fopen(filename, "rb");
86  if (!f)
87  throw std::runtime_error("unable to open file for reading");
88 
89  char tmp[HEADER_LEN];
90  int nread = fread(tmp, sizeof(char), HEADER_LEN, f);
91  if (nread != HEADER_LEN || strcmp(tmp, header))
92  throw std::runtime_error("mal-formed file");
93 
94  len_t m, n;
95  nread = fread(&m, sizeof(len_t), 1, f);
96  nread = fread(&n, sizeof(len_t), 1, f);
97 
98  if (row_first > row_last || row_first < 0 || row_last < 0 || row_first >= m || row_last >= m)
99  throw std::runtime_error("bad first/last row");
100 
101  const len_t m_chunk = row_last - row_first + 1;
102  x.resize(m_chunk, n);
103 
104  for (len_t j=0; j<n; j++)
105  {
106  fseek(f, (row_first + m*j)*sizeof(REAL) + read_offset(), SEEK_SET);
107  nread = fread(x.data_ptr() + m_chunk*j, sizeof(REAL), m_chunk, f);
108  }
109 
110  fclose(f);
111  }
112 
113 
114 
115  namespace
116  {
117  static inline len_t wc_l(FILE *fp, char *buf, const int buflen)
118  {
119  len_t nrows = 0;
120  int readlen = buflen;
121 
122  while (readlen == buflen)
123  {
124  readlen = fread(buf, sizeof(*buf), buflen, fp);
125  char *ptr = buf;
126  char *last = buf + readlen;
127 
128  while ((ptr = (char*)memchr(ptr, '\n', last - ptr)))
129  {
130  ptr++;
131  nrows++;
132  }
133  }
134 
135  return nrows;
136  }
137 
138  static inline len_t nc(FILE *fp, const char sep, char *buf, const int buflen)
139  {
140  len_t ncols = 0;
141  char *ret = fgets(buf, buflen*sizeof(*buf), fp);
142  (void)ret;
143 
144  int readlen = strlen(buf);
145 
146  #pragma omp simd reduction(+:ncols)
147  for (int i=0; i<readlen; i++)
148  ncols += (buf[i]==sep ? 1 : 0);
149 
150  return ncols+1;
151  }
152 
153  template <typename REAL>
154  static inline void read(FILE *fp, char *buf, const int buflen, cpumat<REAL> &x)
155  {
156  const len_t m = x.nrows();
157  const len_t n = x.ncols();
158  REAL *x_d = x.data_ptr();
159 
160  len_t i = 0;
161  while (fgets(buf, buflen*sizeof(*buf), fp) != NULL)
162  {
163  if (i == m)
164  break;
165 
166  char *ptr = buf;
167  char *last;
168 
169  for (len_t j=0; j<n; j++)
170  {
171  double tmp = strtod(ptr, &last);
172  x_d[i + m*j] = (REAL) tmp;
173 
174  ptr = last+1;
175  }
176 
177  i++;
178  }
179  }
180  }
181 
182  template <typename REAL>
183  void read_csv(const char *filename, cpumat<REAL> &x, const char sep=',')
184  {
185  FILE *fp = fopen(filename, "r");
186  if (!fp)
187  throw std::runtime_error("Could not open file");
188 
189  const int buflen = 1024;
190  char *buf = (char*) malloc(buflen * sizeof(*buf));
191  if (buf == NULL)
192  throw std::runtime_error("OOM");
193 
194  len_t m = wc_l(fp, buf, buflen);
195  rewind(fp);
196  len_t n = nc(fp, sep, buf, buflen);
197  rewind(fp);
198 
199  x.resize(m, n);
200  read(fp, buf, buflen, x);
201 
202  fclose(fp);
203  free(buf);
204  }
205 
206  template <typename REAL>
207  void read_csv(const char *filename, cpuvec<REAL> &x)
208  {
209  FILE *fp = fopen(filename, "r");
210  if (!fp)
211  throw std::runtime_error("Could not open file");
212 
213  const int buflen = 1024;
214  char *buf = (char*) malloc(buflen * sizeof(*buf));
215  if (buf == NULL)
216  throw std::runtime_error("OOM");
217 
218  len_t m = wc_l(fp, buf, buflen);
219  rewind(fp);
220  len_t n = 1;
221 
222  x.resize(m);
223  cpumat<REAL> x_mat(x.data_ptr(), m, n, false);
224  read(fp, buf, buflen, x_mat);
225 
226  fclose(fp);
227  free(buf);
228  }
229 }
230 }
231 
232 
233 #endif
fml
Core namespace.
Definition: dimops.hh:10