[Xapian-devel] Re: [Xapian-commits] 7603: trunk/xapian-core/ trunk/xapian-core/backends/flint/ trunk/xapian-core/backends/quartz/

Charlie Hull charlie at juggler.net
Mon Jan 29 17:31:19 GMT 2007


Olly Betts wrote:
> On Tue, Jan 02, 2007 at 03:55:59PM +0000, richard wrote:
>> * backends/quartz/btree.cc,backends/flint/flint_io.h: Patches from
>>   Charlie Hull to allow 2GB+ index files work when compiled using
>>   Visual C++.
> 
> I suspect that xapian-compact.cc (and quartzcompact.cc if you can be
> bothered) will also need fixing since they use off_t.  You need to make
> sure that the stat() function called works for 2GB+ files.  Without this
> compression statistics don't get reported for such files.
> 
> Cheers,
>     Olly
> 

I've produced patches for xapian-compact.cc, quartzcompact.cc and 
quartzcheck.cc (enclosed). Unfortunately I don't have any way of quickly 
testing them here but we should soon have some kind of Xapian test 
framework on Windows, complete with huge files, so I'll do it then.

Charlie
-------------- next part --------------
Index: quartzcheck.cc
===================================================================
--- quartzcheck.cc	(revision 7604)
+++ quartzcheck.cc	(working copy)
@@ -56,6 +56,21 @@
 
 static vector<Xapian::termcount> doclens;
 
+/* define our own stat function, to prevent repeated conditional compilation later */
+#ifdef _MSC_VER
+	#define xapstat __stat64
+	int xapdostat(string spath, struct __stat64 *sbuf )
+	{
+		return _stat64(spath.c_str(), sbuf);
+	}
+#else
+	#define xapstat stat
+	int xapdostat(string spath, struct stat *sbuf )
+	{
+		return stat(spath, sbuf);
+	}	
+#endif
+
 int
 main(int argc, char **argv)
 {
@@ -96,10 +111,10 @@
 
     try {
 	size_t errors = 0;
-	struct stat sb;
+	struct xapstat sb;
 	string record_DB(argv[1]);
 	record_DB += "/record_DB";
-	if (stat(record_DB.c_str(), &sb) == 0) {
+	if (xapdostat(record_DB.c_str(), &sb) == 0) {
 	    // Check a whole quartz database directory.
 	    try {
 		Xapian::Database db(argv[1]);
-------------- next part --------------
Index: xapian-compact.cc
===================================================================
--- xapian-compact.cc	(revision 7604)
+++ xapian-compact.cc	(working copy)
@@ -41,6 +41,14 @@
 
 using namespace std;
 
+#ifdef __WIN32__
+# ifdef _MSC_VER
+// Allow 2GB+ index files
+#  define lseek _lseeki64
+#  define off_t __int64
+# endif
+#endif
+
 #define PROG_NAME "xapian-compact"
 #define PROG_DESC "Compact a flint database, or merge and compact several"
 
@@ -68,6 +76,21 @@
     return key.size() == 1 && key[0] == '\0';
 }
 
+/* define our own stat function, to prevent repeated conditional compilation later */
+#ifdef _MSC_VER
+	#define xapstat __stat64
+	int xapdostat(string spath, struct __stat64 *sbuf )
+	{
+		return _stat64(spath.c_str(), sbuf);
+	}
+#else
+	#define xapstat stat
+	int xapdostat(string spath, struct stat *sbuf )
+	{
+		return stat(spath, sbuf);
+	}	
+#endif
+
 class PostlistCursor : private FlintCursor {
     Xapian::docid offset;
 
@@ -312,9 +335,8 @@
 		     << endl;
 		exit(1);
 	    }
-
-	    struct stat sb;
-	    if (stat(string(srcdir) + "/iamflint", &sb) != 0) {
+	    struct xapstat sb;
+	    if (xapdostat(string(srcdir) + "/iamflint", &sb) != 0) {
 		cout << argv[0] << ": '" << srcdir
 		     << "' is not a flint database directory" << endl;
 		exit(1);
@@ -336,8 +358,8 @@
 	    // exists, but we also get EEXIST if there's an existing file with
 	    // that name.
 	    if (errno == EEXIST) {
-		struct stat sb;
-		if (stat(destdir, &sb) == 0 && S_ISDIR(sb.st_mode))
+		struct xapstat sb;
+		if (xapdostat(destdir, &sb) == 0 && S_ISDIR(sb.st_mode))
 		    errno = 0;
 		else
 		    errno = EEXIST; // stat might have changed it
@@ -386,9 +408,8 @@
 		    s += *t;
 		    s += '.';
 		    tmp.push_back(s);
-
-		    struct stat sb;
-		    if (stat(s + "DB", &sb) == 0)
+		    struct xapstat sb;
+		    if (xapdostat(s + "DB", &sb) == 0)
 			in_size += sb.st_size / 1024;
 		    else
 			bad_stat = true;
@@ -446,9 +467,8 @@
 		    string src(sources[i]);
 		    src += *t;
 		    src += '.';
-
-		    struct stat sb;
-		    if (stat(src + "DB", &sb) == 0) {
+		    struct xapstat sb;
+		    if (xapdostat(src + "DB", &sb) == 0) {
 			if (sb.st_size == 0) continue;
 			in_size += sb.st_size / 1024;
 		    } else {
@@ -500,8 +520,9 @@
 	    out.commit(1);
 
 	    cout << '\r' << *t << ": ";
-	    struct stat sb;
-	    if (!bad_stat && stat(dest + "DB", &sb) == 0) {
+		
+	    struct xapstat sb;
+	    if (!bad_stat && xapdostat(dest + "DB", &sb) == 0) {
 		off_t out_size = sb.st_size / 1024;
 		if (out_size == in_size) {
 		    cout << "Size unchanged (";
-------------- next part --------------
Index: quartzcompact.cc
===================================================================
--- quartzcompact.cc	(revision 7604)
+++ quartzcompact.cc	(working copy)
@@ -47,6 +47,14 @@
 #define PROG_NAME "quartzcompact"
 #define PROG_DESC "Compact a quartz database, or merge and compact several"
 
+#ifdef __WIN32__
+# ifdef _MSC_VER
+// Allow 2GB+ index files
+#  define lseek _lseeki64
+#  define off_t __int64
+# endif
+#endif
+
 static void show_usage() {
     cout << "Usage: "PROG_NAME" [OPTION] SOURCE_DATABASE... DESTINATION_DATABASE\n\n"
 "Options:\n"
@@ -65,6 +73,21 @@
     return key.size() == 1 && key[0] == '\0';
 }
 
+/* define our own stat function, to prevent repeated conditional compilation later */
+#ifdef _MSC_VER
+	#define xapstat __stat64
+	int xapdostat(string spath, struct __stat64 *sbuf )
+	{
+		return _stat64(spath.c_str(), sbuf);
+	}
+	#else
+	#define xapstat stat
+	int xapdostat(string spath, struct stat *sbuf )
+	{
+		return stat(spath, sbuf);
+	}	
+	#endif
+
 class PostlistCursor : private Bcursor {
         Xapian::docid offset;
     public:
@@ -281,8 +304,8 @@
 	// or for some other reason - we also get EEXIST if there's a file
 	// with that name.
 	if (errno == EEXIST) {
-	    struct stat sb;
-	    if (stat(destdir, &sb) == 0 && S_ISDIR(sb.st_mode))
+	    struct xapstat sb;
+	    if (xapdostat(destdir, &sb) == 0 && S_ISDIR(sb.st_mode))
 		errno = 0;
 	    else
 		errno = EEXIST; // stat might have changed it
@@ -358,8 +381,8 @@
 			delete in;
 		    }
 
-		    struct stat sb;
-		    if (stat(src + "DB", &sb) == 0)
+		    struct xapstat sb;
+		    if (xapdostat(src + "DB", &sb) == 0)
 			in_size += sb.st_size / 1024;
 		    else
 			bad_stat = true;
@@ -427,8 +450,8 @@
 			delete in;
 		    }
 
-		    struct stat sb;
-		    if (stat(src + "DB", &sb) == 0)
+		    struct xapstat sb;
+		    if (xapdostat(src + "DB", &sb) == 0)
 			in_size += sb.st_size / 1024;
 		    else
 			bad_stat = true;
@@ -465,8 +488,8 @@
 			delete in;
 		    }
 
-		    struct stat sb;
-		    if (stat(src + "DB", &sb) == 0)
+		    struct xapstat sb;
+		    if (xapdostat(src + "DB", &sb) == 0)
 			in_size += sb.st_size / 1024;
 		    else
 			bad_stat = true;
@@ -528,8 +551,8 @@
 	    out.commit(1);
 
 	    cout << '\r' << *t << ": ";
-	    struct stat sb;
-	    if (!bad_stat && stat(dest + "DB", &sb) == 0) {
+	    struct xapstat sb;
+	    if (!bad_stat && xapdostat(dest + "DB", &sb) == 0) {
 		off_t out_size = sb.st_size / 1024;
 		if (out_size == in_size) {
 		    cout << "Size unchanged (";


More information about the Xapian-devel mailing list