问题描述
我正在使用C程序来处理大量CSV数据。测试文件很小，运行良好。但是，当文件大小增加时，它将开始失败。取决于我是使用gcc还是minGW的gcc进行编译，它会因分段错误或3221225477 / 0xC0000005在不同位置而失败，始终为：
if (fclose(fp)) {
printf("Error closing file: %s,%s,%d.\n",fileName,__func__,__LINE__);
exit(300);
}
请注意，它不会超过fclose（）。或其中之一：
data_PE_T12 = calloc(width_T*dataDepthDay,sizeof(*data_PE_T12));
很长，所以我将尝试展示相关部分。首先是Main函数：
#include <stdio.h>
#include <string.h> // strtok
#include <stdlib.h> // atoi & atof
#include <time.h> // time functions
#include <math.h> // expf()
...
// Array Sizes
static int dataDepth,dataDepthDay;
static int fromTime,toTime;
static int width,width_T,width_H,width_C;
// Array Pointers
static int *timeArray,*timeArrayDay,*timeArrayPE;
static struct sensorHeader_t *headerArray,*headerArray_T,*headerArray_H,*headerArray_C;
// of depth dataDepthDay
static float *data_E_T25,*data_E_T30;
static float *data_E_T12,*data_E_T18,*data_E_H60,*data_E_H70,*data_E_C1500;
static float *data_PE_T12,*data_PE_T18,*data_PE_H60,*data_PE_H70,*data_PE_C1500;
... plus loads more.
// functions
void grabDepth(void); // OK
void grabPayload(void); // OK
... plus loads more.
int main(int argc,char **argv)
{
// Grab Input File Name
if (argc == 2) {
strcpy(rawFile,"in/");
strcat(rawFile,argv[1]);
} else { // dev
strcpy(rawFile,"in/sensor_report.csv");
}
printf("size max = %d",__SIZE_MAX__);
// Parse and copy File
grabDepth();
grabPayload();
// Run functions
genRawData(); // Raw T,H & C files
genExposureE(); //
genExposureAPE(); //
return 0;
}
接下来调用的第一个函数。这将打开主输入文件，并提取许多数组宽度和深度，这些宽度和深度用于对已声明为静态指针的数组进行调用。这样做的想法是，随着文件大小的增加，这将使内存的处理变得美观而灵活。
void grabDepth(void)
{
// 1. Open File
FILE *fp = fopen(rawFile,"r");
char buf[15000]; // Big enough to deal with lots of devices.
if (!fp) {
printf("Can't open the file: %s: %s,rawFile,__LINE__);
exit(100);
}
while (fgets (buf,sizeof(buf),fp)) {
int lineLen = strlen(buf);
int colNum = 1;
char *field = strtok(buf,",");
if (field && strcmp(field,"From") == 0) {
// printf("\n\n*** row 2 ***\n\n");
// int fromTime,toTime = 0;
while (field) {
if (colNum == 2) {
fromTime = atof(field);
}
if (colNum == 4) {
toTime = atof(field);
}
field = strtok(NULL,");
colNum++;
}
// printf("FromTime = %d. ToTime = %d.\n",fromTime,toTime);
dataDepth = ( toTime - fromTime )/900;
// printf("dataDepth = %d.\n",dataDepth);
continue; // to next iteration.
}
// 3. Grab file width from line 10 (commsType) Check if buf Overruns too
if (field && strcmp(field,"TimeStamp") == 0) {
// First Check Line is long enough!
if (lineLen == sizeof(buf)-1) { // buf has overrun!
printf("File Read-Line Overrun: %s,__LINE__);
exit(200);
}
// printf("Line Length = %d\n",lineLen);
// printf("Buf Size = %d\n",sizeof(buf));
width = -2; // ignore timestamps : I ballsed up the commas in csv file (-2 instead of -1)
while (field) {
if(field = strtok(NULL,")) {
width ++;
}
}
break; // out of loop!
}
}
//dataDepthDay = dataDepth/96 + (dataDepth % 96 !=0); // round up?!
dataDepthDay = dataDepth/96; // round down?!
printf("\n 1. grabDepth() Results\n");
printf( "------------------------\n");
printf("Raw Data Width = %d\n",width);
printf("Raw Data Depth = %d\n",dataDepth);
printf("dataDepthDay Depth = %d\n\n",dataDepthDay);
if (fclose(fp)) {
printf("Error closing file: %s,__LINE__);
exit(300);
}
}
之后，它只是依次调用一个函数，所有这些函数都遵循以下一般模式：
void _genRawData(char* sensorType,struct sensorHeader_t *sensorHeader,float *dataArray,int *timeArray,size_t dataDepth,size_t width) {
FILE *fp;
strcpy(fileName,"out/");
strcat(fileName,sensorType);
strcat(fileName,"_raw.csv");
fp = fopen(fileName,"w");
// check file opened OK.
if (fp == NULL) {
printf("Error! Couldn't Create file: %s\n",fileName);
return;
}
printf("building file : %s\n",fileName);
// Allocate Memory
timeArrayDay = calloc(dataDepthDay,sizeof(*timeArrayDay));
timeArrayPE = calloc(dataDepthDay,sizeof(*timeArrayPE)); // xxxx same array as day time array!?
data_E_T12 = calloc(width_T*dataDepthDay,sizeof(*data_E_T12));
data_E_T18 = calloc(width_T*dataDepthDay,sizeof(*data_E_T18));
data_E_H60 = calloc(width_H*dataDepthDay,sizeof(*data_E_H60));
data_E_H70 = calloc(width_H*dataDepthDay,sizeof(*data_E_H70));
// do stuff and build new arrays up and put into files...
if (fclose(fp)) {
printf("Error closing file: %s,__LINE__);
exit(300);
}
}
我在每个2-D数组上只调用一次calloc，为了调试，我删除了free（）调用。
我认为我在内存管理上做错了什么，当数组大小超过某个点时，这让我很痛苦，但是我无法弄清楚出了什么问题。我试图确保我访问的内存已正确分配，并且可以在功能强大的实际计算机（通常是嵌入式人员）上工作，我不希望OS发出数据时会遇到任何问题吗？没有足够的空间！？
解决方法
如果结果对其他人有用。我怀疑calloc和分配的内存的后续使用存在问题。所以我尝试了两件事：
1：检查代码中的内存使用情况：
// Add Values & Write Line on new Day & Reset Accumulator
for (i=0; i < dataDepth; i++) {
for (j=0; j < width; j++) {
if (newDay) {
fprintf(fp,",%.2f",APE_Accum[j]);
data_E_Array[(data_E_Index-1)*width+j] = APE_Accum[j];
if ((data_E_Index-1)*width+j+1 > (width_T*dataDepthDay)) {
printf("Oh bugger...\n");
printf("width_T*dataDepthDay = %d\n",width_T*dataDepthDay);
printf("data_E_Index-1 = %d\n",data_E_Index-1);
printf("width = %d\n",width);
printf("dataDepthDay = %d\n",dataDepthDay);
printf("width_T = %d\n",width_T);
printf("j = %d\n\n",j);
真正凌乱的代码，因此您可以了解我如何失去对数组范围的跟踪。基本上，很明显我搞砸了对愈伤组织大小的计算。我可能会发现这样的问题，但是我认为这不是我的问题的可行答案，因为它可以扩展到更大或更复杂的代码。
2：瓦尔格隆德。遵循@dbush的建议。我移至Ubuntu，安装了Valgrind，然后重新编译...
$ sudo apt install valgrind
$ ps aux | grep-i apt
$ gcc -o graphomatic ./graphomatic.c -lm -g
$ valgrind --leak-check=full --show-leak-kinds=all --verbose --track-origins=yes --log-file=valgrind-log
$ less valgrind-log
鲍勃是你的叔叔。问题跳出来了。我需要添加-lm以链接到数学库。然后使用-g来确保Valgrind输出中包含行号。
==15878== Invalid write of size 4
==15878== at 0x4038EA: _genExposureE (graphomatic.c:867)
==15878== by 0x404A0C: genExposureE (graphomatic.c:1235)
==15878== by 0x400EAA: main (graphomatic.c:122)
==15878== Address 0x75cd604 is 0 bytes after a block of size 660 alloc'd
==15878== at 0x4C2FB55: calloc (in /usr/lib/valgrind/vgpreload_memcheck-amd64-linux.so)
==15878== by 0x404911: genExposureE (graphomatic.c:1222)
==15878== by 0x400EAA: main (graphomatic.c:122)