BRL-CAD
mappedfile.c
Go to the documentation of this file.
1 /* M A P P E D F I L E . C
2  * BRL-CAD
3  *
4  * Copyright (c) 2004-2014 United States Government as represented by
5  * the U.S. Army Research Laboratory.
6  *
7  * This library is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public License
9  * version 2.1 as published by the Free Software Foundation.
10  *
11  * This library is distributed in the hope that it will be useful, but
12  * WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with this file; see the file named COPYING for more
18  * information.
19  */
20 
21 #include "common.h"
22 
23 #include <limits.h> /* for INT_MAX */
24 #include <math.h>
25 #include <string.h>
26 #ifdef HAVE_SYS_TYPES_H
27 # include <sys/types.h>
28 #endif
29 #ifdef HAVE_SYS_STAT_H
30 # include <sys/stat.h>
31 #endif
32 #ifdef HAVE_SYS_MMAN_H
33 # include <sys/mman.h>
34 # if !defined(MAP_FAILED)
35 # define MAP_FAILED ((void *)-1) /* Error return from mmap() */
36 # endif
37 #endif
38 #include "bio.h"
39 
40 #include "bu/debug.h"
41 #include "bu/file.h"
42 #include "bu/list.h"
43 #include "bu/log.h"
44 #include "bu/malloc.h"
45 #include "bu/mapped_file.h"
46 #include "bu/parallel.h"
47 #include "bu/str.h"
48 #include "bu/vls.h"
49 
50 
51 /* list of currently open mapped files */
52 static struct bu_list bu_mapped_file_list = BU_LIST_INIT_ZERO;
53 
54 
55 struct bu_mapped_file *
56 bu_open_mapped_file(const char *name, const char *appl)
57 /* file name */
58 /* non-null only when app. will use 'apbuf' */
59 {
60  struct bu_mapped_file *mp = (struct bu_mapped_file *)NULL;
61  char *real_path = bu_realpath(name, NULL);
62 #ifdef HAVE_SYS_STAT_H
63  struct stat sb;
64  int fd = -1; /* unix file descriptor */
65  int readval;
66  ssize_t bytes_to_go, nbytes;
67 #else
68  FILE *fp = (FILE *)NULL; /* stdio file pointer */
69 #endif
70  int ret;
71 
73  bu_log("bu_open_mapped_file(%s(canonical path - %s), %s)\n", name, real_path, appl?appl:"(NIL)");
74 
75  /* See if file has already been mapped, and can be shared */
77  if (!BU_LIST_IS_INITIALIZED(&bu_mapped_file_list)) {
78  BU_LIST_INIT(&bu_mapped_file_list);
79  }
80 
81  for (BU_LIST_FOR(mp, bu_mapped_file, &bu_mapped_file_list)) {
83 
84  /* find a match */
85 
86  if (!BU_STR_EQUAL(real_path, mp->name))
87  continue;
88  if (appl && !BU_STR_EQUAL(appl, mp->appl))
89  continue;
90 
91  /* found a match */
92 
93  /* if mapped file still exists, verify size and modtime */
94  if (!mp->dont_restat) {
95 
97  fd = open(real_path, O_RDONLY | O_BINARY);
99 
100  /* If file didn't vanish from disk, make sure it's the same file */
101  if (fd >= 0) {
102 
103 #ifdef HAVE_SYS_STAT_H
105  ret = fstat(fd, &sb);
107 
108  if (ret < 0) {
109  /* odd, open worked but fstat failed. assume it
110  * vanished from disk and the mapped copy is still
111  * OK.
112  */
113 
115  (void)close(fd);
117  fd = -1;
118  }
119  if ((size_t)sb.st_size != mp->buflen) {
120  bu_log("bu_open_mapped_file(%s) WARNING: File size changed from %ld to %ld, opening new version.\n", real_path, mp->buflen, sb.st_size);
121  /* mp doesn't reflect the file any longer. Invalidate. */
122  mp->appl = bu_strdup("__STALE__");
123  /* Can't invalidate old copy, it may still be in use. */
124  break;
125  }
126  if (sb.st_mtime != mp->modtime) {
127  bu_log("bu_open_mapped_file(%s) WARNING: File modified since last mapped, opening new version.\n", real_path);
128  /* mp doesn't reflect the file any longer. Invalidate. */
129  mp->appl = bu_strdup("__STALE__");
130  /* Can't invalidate old copy, it may still be in use. */
131  break;
132  }
133  /* To be completely safe, should check st_dev and st_inum */
134 #endif
135  }
136 
137  /* It is safe to reuse mp */
138  mp->uses++;
139 
141  if (UNLIKELY(bu_debug&BU_DEBUG_MAPPED_FILE))
142  bu_pr_mapped_file("open_reused", mp);
143 
144  return mp;
145  }
146 
147  /* It is safe to reuse mp */
148  mp->uses++;
149  return mp;
150 
151  }
152  /* done iterating over mapped file list */
154 
155  /* necessary in case we take a 'fail' path before BU_ALLOC() */
156  mp = NULL;
157 
158  /* File is not yet mapped or has changed, open file read only if
159  * we didn't find it earlier.
160  */
161 #ifdef HAVE_SYS_STAT_H
162  if (fd < 0) {
164  fd = open(real_path, O_RDONLY | O_BINARY);
166  }
167 
168  if (UNLIKELY(fd < 0)) {
169  if (UNLIKELY(bu_debug&BU_DEBUG_MAPPED_FILE))
170  perror(real_path);
171  goto fail;
172  }
173 
175  ret = fstat(fd, &sb);
177 
178  if (UNLIKELY(ret < 0)) {
179  perror(real_path);
180  goto fail;
181  }
182 
183  if (UNLIKELY(sb.st_size == 0)) {
184  bu_log("bu_open_mapped_file(%s) 0-length file\n", real_path);
185  goto fail;
186  }
187 #endif /* HAVE_SYS_STAT_H */
188 
189  /* Optimistically assume that things will proceed OK */
190  BU_ALLOC(mp, struct bu_mapped_file);
191  mp->name = bu_strdup(real_path);
192  if (appl) mp->appl = bu_strdup(appl);
193 
194 #ifdef HAVE_SYS_STAT_H
195  mp->buflen = sb.st_size;
196  mp->modtime = sb.st_mtime;
197 # ifdef HAVE_SYS_MMAN_H
198 
199  /* Attempt to access as memory-mapped file */
201  mp->buf = mmap(NULL, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
203 
204  if (UNLIKELY(mp->buf == MAP_FAILED))
205  perror(real_path);
206 
207  if (mp->buf != MAP_FAILED) {
208  /* OK, its memory mapped in! */
209  mp->is_mapped = 1;
210  /* It's safe to close the fd now, the manuals say */
211  } else
212 # endif /* HAVE_SYS_MMAN_H */
213  {
214  /* Allocate a local zero'd buffer, and slurp it in always
215  * leaving space for a trailing zero.
216  */
217  mp->buf = bu_calloc(1, sb.st_size+1, real_path);
218 
219  nbytes = 0;
220  bytes_to_go = sb.st_size;
222  while (nbytes < sb.st_size) {
223  readval = read(fd, ((char *)(mp->buf)) + nbytes, ((bytes_to_go > INT_MAX) ? (INT_MAX) : (bytes_to_go)));
224  if (UNLIKELY(readval < 0)) {
226  perror(real_path);
227  bu_free(mp->buf, real_path);
228  goto fail;
229  } else {
230  nbytes += readval;
231  bytes_to_go -= readval;
232  }
233  }
235 
236  if (UNLIKELY(nbytes != sb.st_size)) {
237  perror(real_path);
238  bu_free(mp->buf, real_path);
239  goto fail;
240  }
241  }
242 
243 #else /* !HAVE_SYS_STAT_H */
244 
245  /* Read it in with stdio, with no clue how big it is */
247  fp = fopen(real_path, "rb");
249 
250  if (UNLIKELY(fp == NULL)) {
251  perror(real_path);
252  goto fail;
253  }
254  /* Read it once to see how large it is */
255  {
256  char buf[32768] = {0};
257  int got;
258  mp->buflen = 0;
259 
261  while ((got = fread(buf, 1, sizeof(buf), fp)) > 0)
262  mp->buflen += got;
263  rewind(fp);
265 
266  }
267  /* Allocate the necessary buffer */
268  mp->buf = bu_calloc(1, mp->buflen+1, real_path);
269 
270  /* Read it again into the buffer */
272  ret = fread(mp->buf, mp->buflen, 1, fp);
274 
275  if (UNLIKELY(ret != 1)) {
277  perror("fread");
278  fclose(fp);
280 
281  bu_log("bu_open_mapped_file() 2nd fread failed? len=%d\n", mp->buflen);
282  bu_free(mp->buf, "non-unix fread buf");
283  goto fail;
284  }
285 
287  fclose(fp);
289 #endif
290 
291  if (fd >= 0) {
293  (void)close(fd);
295  }
296 
297  mp->uses = 1;
299 
301  BU_LIST_APPEND(&bu_mapped_file_list, &mp->l);
303 
304  if (UNLIKELY(bu_debug&BU_DEBUG_MAPPED_FILE)) {
305  bu_pr_mapped_file("1st_open", mp);
306  }
307  if (real_path) {
308  bu_free(real_path, "real_path alloc from bu_realpath");
309  }
310  return mp;
311 
312 fail:
313  if (fd >= 0) {
315  (void)close(fd);
317  }
318 
319  if (mp) {
320  bu_free(mp->name, "mp->name");
321  if (mp->appl) bu_free(mp->appl, "mp->appl");
322  /* Don't free mp->buf here, it might not be bu_malloced but mmaped */
323  bu_free(mp, "mp from bu_open_mapped_file fail");
324  }
325 
326  if (UNLIKELY(bu_debug&BU_DEBUG_MAPPED_FILE))
327  bu_log("bu_open_mapped_file(%s, %s) can't open file\n",
328  real_path, appl ? appl: "(NIL)");
329 
330  if (real_path) {
331  bu_free(real_path, "real_path alloc from bu_realpath");
332  }
333 
334  return (struct bu_mapped_file *)NULL;
335 }
336 
337 
338 void
340 {
341  if (UNLIKELY(!mp)) {
342  bu_log("bu_close_mapped_file() called with null pointer\n");
343  return;
344  }
345 
346  BU_CK_MAPPED_FILE(mp);
347 
349  bu_pr_mapped_file("close:uses--", mp);
350 
352  --mp->uses;
354 }
355 
356 
357 void
358 bu_pr_mapped_file(const char *title, const struct bu_mapped_file *mp)
359 {
360  if (UNLIKELY(!mp))
361  return;
362 
363  BU_CK_MAPPED_FILE(mp);
364 
365  bu_log("%p mapped_file %s %p len=%ld mapped=%d, uses=%d %s\n",
366  (void *)mp, mp->name, mp->buf, mp->buflen,
367  mp->is_mapped, mp->uses,
368  title);
369 }
370 
371 
372 void
374 {
375  struct bu_mapped_file *mp, *next;
376 
378  bu_log("bu_free_mapped_files(verbose=%d)\n", verbose);
379 
381 
382  next = BU_LIST_FIRST(bu_mapped_file, &bu_mapped_file_list);
383  while (BU_LIST_NOT_HEAD(next, &bu_mapped_file_list)) {
384  BU_CK_MAPPED_FILE(next);
385  mp = next;
386  next = BU_LIST_NEXT(bu_mapped_file, &mp->l);
387 
388  if (mp->uses > 0) continue;
389 
390  /* Found one that needs to have storage released */
391  if (UNLIKELY(verbose || (bu_debug&BU_DEBUG_MAPPED_FILE)))
392  bu_pr_mapped_file("freeing", mp);
393 
394  BU_LIST_DEQUEUE(&mp->l);
395 
396  /* If application pointed mp->apbuf at mp->buf, break that
397  * association so we don't double-free the buffer.
398  */
399  if (mp->apbuf == mp->buf) mp->apbuf = (void *)NULL;
400 
401 #ifdef HAVE_SYS_MMAN_H
402  if (mp->is_mapped) {
403  int ret;
405  ret = munmap(mp->buf, (size_t)mp->buflen);
407 
408  if (UNLIKELY(ret < 0))
409  perror("munmap");
410 
411  /* XXX How to get this chunk of address space back to malloc()? */
412  } else
413 #endif
414  {
415  bu_free(mp->buf, "bu_mapped_file.buf[]");
416  }
417  mp->buf = (void *)NULL; /* sanity */
418  bu_free((void *)mp->name, "bu_mapped_file.name");
419 
420  if (mp->appl)
421  bu_free((void *)mp->appl, "bu_mapped_file.appl");
422 
423  bu_free((void *)mp, "struct bu_mapped_file");
424  }
426 }
427 
428 
429 struct bu_mapped_file *
430 bu_open_mapped_file_with_path(char *const *path, const char *name, const char *appl)
431 
432 /* file name */
433 /* non-null only when app. will use 'apbuf' */
434 {
435  char * const *pathp = path;
436  struct bu_vls str = BU_VLS_INIT_ZERO;
437  struct bu_mapped_file *ret;
438 
439  BU_ASSERT_PTR(name, !=, NULL);
440  BU_ASSERT_PTR(pathp, !=, NULL);
441 
442  /* Do not resort to path for a rooted filename */
443  if (name[0] == '/')
444  return bu_open_mapped_file(name, appl);
445 
446  /* Try each path prefix in sequence */
447  for (; *pathp != NULL; pathp++) {
448  bu_vls_strcpy(&str, *pathp);
449  bu_vls_putc(&str, '/');
450  bu_vls_strcat(&str, name);
451 
452  ret = bu_open_mapped_file(bu_vls_addr(&str), appl);
453  if (ret) {
454  bu_vls_free(&str);
455  return ret;
456  }
457  }
458 
459  /* Failure, none of the opens succeeded */
460  bu_vls_free(&str);
461  return (struct bu_mapped_file *)NULL;
462 }
463 
464 /*
465  * Local Variables:
466  * mode: C
467  * tab-width: 8
468  * indent-tabs-mode: t
469  * c-file-style: "stroustrup"
470  * End:
471  * ex: shiftwidth=4 tabstop=8
472  */
ptrdiff_t ssize_t
Definition: common.h:119
#define BU_LIST_FOR(p, structure, hp)
Definition: list.h:365
void bu_log(const char *,...) _BU_ATTR_PRINTF12
Definition: log.c:176
Definition: list.h:118
Definition: clone.c:90
void bu_vls_strcat(struct bu_vls *vp, const char *s)
Definition: vls.c:368
void bu_semaphore_acquire(unsigned int i)
Definition: semaphore.c:180
#define BU_LIST_IS_INITIALIZED(_hp)
Definition: list.h:175
#define BU_LIST_INIT_ZERO
Definition: list.h:167
void bu_pr_mapped_file(const char *title, const struct bu_mapped_file *mp)
Definition: mappedfile.c:358
Header file for the BRL-CAD common definitions.
#define BU_LIST_APPEND(old, new)
Definition: list.h:197
time_t modtime
Definition: mapped_file.h:91
#define BU_SEM_MAPPEDFILE
Definition: parallel.h:181
void bu_vls_free(struct bu_vls *vp)
Definition: vls.c:248
#define BU_ALLOC(_ptr, _type)
Definition: malloc.h:223
void * bu_calloc(size_t nelem, size_t elsize, const char *str)
Definition: malloc.c:321
#define BU_CK_MAPPED_FILE(_mf)
Definition: mapped_file.h:101
struct bu_mapped_file * bu_open_mapped_file(const char *name, const char *appl)
Definition: mappedfile.c:56
char * bu_vls_addr(const struct bu_vls *vp)
Definition: vls.c:111
uint32_t magic
Magic # for mem id/check.
Definition: list.h:119
struct bu_list l
Definition: mapped_file.h:83
void bu_semaphore_release(unsigned int i)
Definition: semaphore.c:218
#define BU_MAPPED_FILE_MAGIC
Definition: magic.h:57
int bu_debug
Definition: globals.c:87
#define BU_LIST_INIT(_hp)
Definition: list.h:148
#define BU_ASSERT_PTR(_lhs, _relation, _rhs)
Definition: defines.h:227
#define BU_SEM_SYSCALL
Definition: parallel.h:178
void bu_free_mapped_files(int verbose)
Definition: mappedfile.c:373
void bu_close_mapped_file(struct bu_mapped_file *mp)
Definition: mappedfile.c:339
void bu_vls_strcpy(struct bu_vls *vp, const char *s)
Definition: vls.c:310
void bu_free(void *ptr, const char *str)
Definition: malloc.c:328
#define BU_VLS_INIT_ZERO
Definition: vls.h:84
#define BU_LIST_DEQUEUE(cur)
Definition: list.h:209
#define BU_DEBUG_MAPPED_FILE
Definition: debug.h:66
HIDDEN void verbose(struct human_data_t *dude)
Definition: human.c:2008
char * bu_realpath(const char *path, char *resolved_path)
Definition: realpath.c:33
Definition: vls.h:56
void bu_vls_putc(struct bu_vls *vp, int c)
Definition: vls.c:666
#define BU_LIST_NEXT(structure, hp)
Definition: list.h:316
#define BU_LIST_NOT_HEAD(p, hp)
Definition: list.h:324
struct bu_mapped_file * bu_open_mapped_file_with_path(char *const *path, const char *name, const char *appl)
Definition: mappedfile.c:430
#define BU_LIST_FIRST(structure, hp)
Definition: list.h:312
#define bu_strdup(s)
Definition: str.h:71
#define UNLIKELY(expression)
Definition: common.h:282
#define BU_STR_EQUAL(s1, s2)
Definition: str.h:126