//purpose: 生成随机的不重复的测试数据
//1000w数据量，要保证生成不重复的数据量，一般的程序没有做到。但，本程序做到了。
#include <time.h>
#include <assert.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <bitset>
#include <iostream>
#include <time.h>
#define BITSPERWORD 32
#define SHIFT 5
#define MASK 0x1F
#define N_SIZE 10000000
const unsigned int CLK_TCK = 1000;
using namespace std;
int random_num[N_SIZE] = {0};
const int max_each_scan = N_SIZE/2;
unsigned char *g_bitmap = NULL;
static int g_size = 0;
static int g_base = 0;
int qsort_num[N_SIZE] = {0};
int sort_num[N_SIZE] = {0};
int sort2_num[N_SIZE] = {0};
void swap(int* a, int* b)//第一种，也是最常用的一种
{
int t;
t=*a;
*a=*b;
*b=t;
}
void create_data_file()
{
int n;
int i, j;
FILE *fp = fopen("data.txt", "w");
assert(fp);
for (n = 0; n < N_SIZE; n++) //之前此处写成了n=0;n<size。导致下面有一段小程序的测试数据出现了0，特此订正。
random_num[n] = n+1;
srand((unsigned)time(NULL));
printf("random_num init ok\n");
for (i = N_SIZE - 1; i >= 1; --i)
{
swap(&random_num[i], &random_num[rand() % i]);
}
for (n = 0; n < N_SIZE; n++)
fprintf(fp, "%d ", random_num[n]);
fclose(fp);
}
int bitmap_init(int size, int start)
{
g_bitmap = (unsigned char *)malloc((size/8+1)*sizeof(char));
if(g_bitmap == NULL)
return 0;
g_base = start;
g_size = size/8+1;
memset(g_bitmap, 0x0, g_size);
return 1;
}
void bitmap_clr()
{
if(NULL != g_bitmap && g_size != 0)
memset(g_bitmap, 0x0, g_size);
}
int bitmap_set(int index)
{
int quo = (index-g_base)/8 ;
int remainder = (index-g_base)%8;
unsigned char x = (0x1<<remainder);
if( quo > g_size)
return 0;
g_bitmap[quo] |= x;
return 1;
}
int bitmap_get(int i)
{
int quo = (i)/8 ;
int remainder = (i)%8;
unsigned char x = (0x1<<remainder);
unsigned char res;
if( quo > g_size)
return -1;
res = g_bitmap[quo] & x;
return res > 0 ? 1 : 0;
}
int bitmap_data(int index)
{
return (index + g_base);
}
int bitmap_free()
{
free(g_bitmap);
g_bitmap = NULL;
g_size = 0;
g_base = 0;
}
void my_sort_num222()
{
FILE *fp_unsort_file = NULL;
int i;
int num;
clock_t begin = clock();
bitmap_init(max_each_scan,0);
fp_unsort_file = fopen("data.txt", "r");
assert(fp_unsort_file);
// the first time scan to sort the data between 0 - 4999999
while (fscanf(fp_unsort_file, "%d ", &num) != EOF)
{
if (num < max_each_scan)
bitmap_set(num);
}
// write the sorted data into file
for (i = 0; i < max_each_scan; i++)
{
if (bitmap_get(i) == 1)
sort2_num[i] = i;
}
// the second time scan to sort the data between 5000000 - 9999999
int result = fseek(fp_unsort_file, 0, SEEK_SET);
if (result)
printf("fseek failed!\n");
else
{
bitmap_clr();
while (fscanf(fp_unsort_file, "%d ", &num) != EOF)
{
if (num >= max_each_scan && num < 10000000)
{
num -= max_each_scan;
bitmap_set(num);
}
}
for (i = 0; i < max_each_scan; i++)
{
if (bitmap_get(i) == 1)
sort2_num[i + max_each_scan] = i + max_each_scan;
}
}
fclose(fp_unsort_file);
clock_t end = clock();
printf("sort2 consum %d ms\n",(end - begin) / CLK_TCK);;
}
void create_sort_file222()
{
FILE *fp_sort_file = NULL;
FILE *fp_unsort_file = NULL;
int i;
int num;
bitmap_init(max_each_scan,0);
fp_unsort_file = fopen("data.txt", "r");
assert(fp_unsort_file);
// the first time scan to sort the data between 0 - 4999999
while (fscanf(fp_unsort_file, "%d ", &num) != EOF)
{
if (num < max_each_scan)
bitmap_set(num);
}
fp_sort_file = fopen("sort2.txt", "w");
assert(fp_sort_file);
// write the sorted data into file
for (i = 0; i < max_each_scan; i++)
{
if (bitmap_get(i) == 1)
fprintf(fp_sort_file, "%d ", i);
}
// the second time scan to sort the data between 5000000 - 9999999
int result = fseek(fp_unsort_file, 0, SEEK_SET);
if (result)
printf("fseek failed!\n");
else
{
bitmap_clr();
while (fscanf(fp_unsort_file, "%d ", &num) != EOF)
{
if (num >= max_each_scan && num < 10000000)
{
num -= max_each_scan;
bitmap_set(num);
}
}
for (i = 0; i < max_each_scan; i++)
{
if (bitmap_get(i) == 1)
fprintf(fp_sort_file, "%d ", i + max_each_scan);
}
}
fclose(fp_sort_file);
fclose(fp_unsort_file);
}
void my_sort_num()
{
FILE *fp_unsort_file = NULL;
int i;
int num;
clock_t begin = clock();
bitset<max_each_scan> bit_map;
bit_map.reset();
fp_unsort_file = fopen("data.txt", "r");
assert(fp_unsort_file);
// the first time scan to sort the data between 0 - 4999999
while (fscanf(fp_unsort_file, "%d ", &num) != EOF)
{
if (num < max_each_scan)
bit_map.set(num, 1);
}
// write the sorted data into file
for (i = 0; i < max_each_scan; i++)
{
if (bit_map[i] == 1)
sort_num[i] = i;
}
// the second time scan to sort the data between 5000000 - 9999999
int result = fseek(fp_unsort_file, 0, SEEK_SET);
if (result)
printf("fseek failed!\n");
else
{
bit_map.reset();
while (fscanf(fp_unsort_file, "%d ", &num) != EOF)
{
if (num >= max_each_scan && num < 10000000)
{
num -= max_each_scan;
bit_map.set(num, 1);
}
}
for (i = 0; i < max_each_scan; i++)
{
if (bit_map[i] == 1)
sort_num[i + max_each_scan] = i + max_each_scan;
}
}
fclose(fp_unsort_file);
clock_t end = clock();
printf("sort1 consum %d ms\n",(end - begin) / CLK_TCK);;
}
void create_sort_file()
{
FILE *fp_sort_file = NULL;
FILE *fp_unsort_file = NULL;
int i;
int num;
bitset<max_each_scan> bit_map;
bit_map.reset();
fp_unsort_file = fopen("data.txt", "r");
assert(fp_unsort_file);
// the first time scan to sort the data between 0 - 4999999
while (fscanf(fp_unsort_file, "%d ", &num) != EOF)
{
if (num < max_each_scan)
bit_map.set(num, 1);
}
fp_sort_file = fopen("sort.txt", "w");
assert(fp_sort_file);
// write the sorted data into file
for (i = 0; i < max_each_scan; i++)
{
if (bit_map[i] == 1)
fprintf(fp_sort_file, "%d ", i);
}
// the second time scan to sort the data between 5000000 - 9999999
int result = fseek(fp_unsort_file, 0, SEEK_SET);
if (result)
printf("fseek failed!\n");
else
{
bit_map.reset();
while (fscanf(fp_unsort_file, "%d ", &num) != EOF)
{
if (num >= max_each_scan && num < 10000000)
{
num -= max_each_scan;
bit_map.set(num, 1);
}
}
for (i = 0; i < max_each_scan; i++)
{
if (bit_map[i] == 1)
fprintf(fp_sort_file, "%d ", i + max_each_scan);
}
}
fclose(fp_sort_file);
fclose(fp_unsort_file);
}
int intcomp(int *x,int *y){
return *x-*y;
}
int myintcomp(const void *x, const void *y)
{
int *m = (int*)x;
int *n = (int*)y;
return *m - *n;
}
void my_qsort_num()
{
FILE *fp_unsort_file = NULL;
int i=0;
int num;
clock_t begin = clock();
fp_unsort_file = fopen("data.txt", "r");
assert(fp_unsort_file);
// the first time scan to sort the data between 0 - 4999999
while (fscanf(fp_unsort_file, "%d ", &num) != EOF)
{
qsort_num[i++] = num;
}
qsort((void*)qsort_num,N_SIZE,sizeof(int),myintcomp);
fclose(fp_unsort_file);
clock_t end = clock();
printf("qsort consum %d ms\n",(end - begin) / CLK_TCK);;
}
int main()
{
//create_data_file();
//create_sort_file();
//create_sort_file222();
int x = 5, y = 4,z = 5,q = 6;
printf("intcomp=%d\n",myintcomp((const void *)&x, (const void *)&y));
printf("intcomp=%d\n",myintcomp((const void *)&x, (const void *)&z));
printf("intcomp=%d\n",myintcomp((const void *)&x, (const void *)&q));
printf("intcomp=%d\n",intcomp(&x, &y));
printf("intcomp=%d\n",intcomp(&x, &z));
printf("intcomp=%d\n",intcomp(&x, &q));
my_sort_num222();
my_qsort_num();
my_sort_num();
return 0;
}
第一节、如何给磁盘文件排序
问题描述：
输入：一个最多含有n个不重复的正整数（也就是说可能含有少于n个不重复正整数）的文件，其中每个数都小于等于n，且n=10^7。
输出：得到按从小到大升序排列的包含所有输入的整数的列表。
条件：最多有大约1MB的内存空间可用，但磁盘空间足够。且要求运行时间在5分钟以下，10秒为最佳结果。
分析：下面咱们来一步一步的解决这个问题，
1、归并排序。你可能会想到把磁盘文件进行归并排序，但题目要求你只有1MB的内存空间可用，所以，归并排序这个方法不行。
2、位图方案。熟悉位图的朋友可能会想到用位图来表示这个文件集合。例如正如编程珠玑一书上所述，用一个20位长的字符串来表示一个所有元素都小于20的简单的非负整数集合，边框用如下字符串来表示集合{1,2,3,5,8,13}：
0 1 1 1 0 1 0 0 1 0 0 0 0 1 0 0 0 0 0 0
上述集合中各数对应的位置则置1，没有对应的数的位置则置0。
参考编程珠玑一书上的位图方案，针对我们的10^7个数据量的磁盘文件排序问题，我们可以这么考虑，由于每个7位十进制整数表示一个小于1000万的整数。我们可以使用一个具有1000万个位的字符串来表示这个文件，其中，当且仅当整数i在文件中存在时，第i位为1。采取这个位图的方案是因为我们面对的这个问题的特殊性：1、输入数据限制在相对较小的范围内，2、数据没有重复，3、其中的每条记录都是单一的整数，没有任何其它与之关联的数据。
所以，此问题用位图的方案分为以下三步进行解决：
- 第一步，将所有的位都置为0，从而将集合初始化为空。
- 第二步，通过读入文件中的每个整数来建立集合，将每个对应的位都置为1。
- 第三步，检验每一位，如果该位为1，就输出对应的整数。
1096
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
