//----------------------------------------------------------------------------- // Copyright © 2004 - Philip Howard - All rights reserved // // This program is free software; you can redistribute it and/or // modify it under the terms of the GNU General Public License // as published by the Free Software Foundation; either version 2 // of the License, or (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. //----------------------------------------------------------------------------- // package libh/weblogsplit // homepage http://libh.slashusr.org/ //----------------------------------------------------------------------------- // author Philip Howard // email phil at ipal dot org // homepage http://phil.ipal.org/ //----------------------------------------------------------------------------- // This file is best viewed using a fixed spaced font such as Courier // and in a display at least 112 columns wide. //----------------------------------------------------------------------------- #include #include #include #include #include #include #include #include #include #include #include #include //----------------------------------------------------------------------------- // configuration //----------------------------------------------------------------------------- #define FIFO_POLL_TIMEOUT 20000 #define NAME_SEP_CHAR '|' #define PIPE_BLOCK_INTERVAL 20 //----------------------------------------------------------------------------- // global variables //----------------------------------------------------------------------------- static unsigned long long total_bytes = 0; static unsigned long long total_lines = 0; static unsigned long long total_items = 0; static unsigned long total_files = 0; static unsigned long total_opens = 0; static AVL tree_by_name ; static AVL tree_by_time ; static vrb_p read_buffer = NULL; static char * argv_name = NULL; static char * show_bytes_ptr = NULL; static char * show_lines_ptr = NULL; static char * show_items_ptr = NULL; static char * show_files_ptr = NULL; static char * show_opens_ptr = NULL; static size_t show_bytes_len = 0; static size_t show_lines_len = 0; static size_t show_items_len = 0; static size_t show_files_len = 0; static size_t show_opens_len = 0; static size_t page_size = 0; static int ruid = 0; static int euid = 0; static int rgid = 0; static int egid = 0; static int max_files ;// configured maximum files open static int idle_time ;// no fd may be idle longer than this static int threshold ;// above this level, close sooner (1..10000) static int num_open ;// how many fds currently now open static int clean_cycle ;// period for cleanup of open files //----------------------------------------------------------------------------- // struct fd_node (with fd_name and fd_time) // // purpose Hold a node for a collection which represents all currently // open file descriptors. These nodes are organized into two // parallel AVL trees, one indexed by the file name so a search // can find the file descriptor to see if a file is already open, // and the other indexed by the time last written, in order to // select file descriptors not recently used for closing. //----------------------------------------------------------------------------- struct fd_name { avl_link link ; char * key ; }; struct fd_time { avl_link link ; time_t key ; }; struct fd_node { struct fd_name name ; struct fd_time time ; int fd ; char fname [1] ; }; //----------------------------------------------------------------------------- // function fd_cmp_name // // purpose Method for AVL tree to compare file names between nodes. // // arguments 1 (struct fd_name *) pointer to link struct in node one // 2 (struct fd_name *) pointer to link struct in node two // 3 (AVL *) pointer to AVL tree // // returns (int) comparison results: // (int) < 0 : node one < node two // (int) == 0 : node one == node two // (int) > 0 : node one > node two //----------------------------------------------------------------------------- static int fd_cmp_name ( avl_link * arg_link_one , avl_link * arg_link_two , AVL * arg_avl ) { return strcmp( ((struct fd_name *)arg_link_one)->key, ((struct fd_name *)arg_link_two)->key ); } //----------------------------------------------------------------------------- // function fd_cmp_time // // purpose Method for AVL tree to compare last write time between nodes. // // arguments 1 (struct fd_time *) pointer to link struct in node one // 2 (struct fd_time *) pointer to link struct in node two // 3 (AVL *) pointer to AVL tree // // returns (int) comparison results: // (int) < 0 : node one < node two // (int) == 0 : node one == node two // (int) > 0 : node one > node two //----------------------------------------------------------------------------- static int fd_cmp_time ( avl_link * arg_link_one , avl_link * arg_link_two , AVL * arg_avl ) { return ((struct fd_time *)arg_link_one)->key < ((struct fd_time *)arg_link_two)->key ? -1 : ((struct fd_time *)arg_link_one)->key > ((struct fd_time *)arg_link_two)->key ? 1 : 0; } //----------------------------------------------------------------------------- // function update_totals // // purpose Update the totals in the process title. // // arguments -none- // // returns (void) //----------------------------------------------------------------------------- static void update_totals () { if ( show_bytes_ptr ) { snprintf( show_bytes_ptr, show_bytes_len + 1, "B=%0*llu", (int) ( show_bytes_len - 2 ), total_bytes ); } if ( show_lines_ptr ) { snprintf( show_lines_ptr, show_lines_len + 1, "L=%0*llu", (int) ( show_lines_len - 2 ), total_lines ); } if ( show_items_ptr ) { snprintf( show_items_ptr, show_items_len + 1, "I=%0*llu", (int) ( show_items_len - 2 ), total_items ); } if ( show_files_ptr ) { snprintf( show_files_ptr, show_files_len + 1, "F=%0*lu", (int) ( show_files_len - 2 ), total_files ); } if ( show_opens_ptr ) { snprintf( show_opens_ptr, show_opens_len + 1, "O=%0*lu", (int) ( show_opens_len - 2 ), total_opens ); } return; } //----------------------------------------------------------------------------- // process run_fifo_buffer // // purpose Act as a buffering process to queue data coming from the web // server before going to the logger process to minimize blocking // the web server. // // arguments 1 (size_t) minimum size of buffer to use // // returns (int) == -1 : error // (int) == 0 : end of file //----------------------------------------------------------------------------- int run_fifo_buffer ( size_t arg_fifo_size ) { struct pollfd poll_list [2]; struct pollfd * poll_p ; vrb_p fifo_buffer ; time_t block_time ; time_t block_next ; time_t block_secs ; int poll_n ; int try_again ; int read_eof ; //--------------------------------------------------- // Do not show any proctitle totals except for bytes. //--------------------------------------------------- show_lines_ptr = NULL; show_items_ptr = NULL; show_files_ptr = NULL; show_opens_ptr = NULL; update_totals(); //---------------------------- // Create virtual ring buffer. //---------------------------- if ( ! ( fifo_buffer = vrb_new( arg_fifo_size, NULL ) ) ) { msg_eprintf__exit( 1, "error creating buffer of %Zu bytes: %s", arg_fifo_size, strerror( errno ) ); } //-------------------------------------- // Set file descriptors to non-blocking. //-------------------------------------- if ( fcntl( 0, F_SETFL, O_NONBLOCK ) < 0 ) { msg_eprintf__exit( 1, "error setting O_NONBLOCK on %sput fd %d: %s", "in", 0, strerror( errno ) ); } if ( fcntl( 1, F_SETFL, O_NONBLOCK ) < 0 ) { msg_eprintf__exit( 1, "error setting O_NONBLOCK on %sput fd %d: %s", "out", 1, strerror( errno ) ); } //---------------------------------- // Initialize check of blocked pipe. //---------------------------------- block_time = 0; block_next = 0; block_secs = 0; //-------------------------------- // Initialize for poll event loop. //-------------------------------- poll_list[0].fd = 0; poll_list[0].events = POLLIN; poll_list[1].fd = 1; poll_list[1].events = POLLOUT; read_eof = 0; //------------------------------------------------------------ // Do main I/O loop until input has ended and buffer is empty. //------------------------------------------------------------ for (;;) { poll_p = poll_list + 1; poll_n = 0; try_again = 0; //---------- // Try read. //---------- if ( ! read_eof && vrb_space_len( fifo_buffer ) >= page_size ) { ssize_t read_len; read_len = read( 0, vrb_space_ptr( fifo_buffer ), vrb_space_len( fifo_buffer ) ); if ( read_len < 0 ) { if ( errno == EAGAIN ) { -- poll_p; ++ poll_n; } else { msg_eprintf__exit( 1, "error in %s()", "read", strerror( errno ) ); } } else if ( read_len == 0 ) { close( 0 ); read_eof = 1; } else { vrb_give( fifo_buffer, read_len ); try_again = 1; total_bytes = vrb_data_len( fifo_buffer ); update_totals(); } } //----------- // Try write. //----------- if ( vrb_data_len( fifo_buffer ) > 0 ) { ssize_t write_len; write_len = write( 1, vrb_data_ptr( fifo_buffer ), vrb_data_len( fifo_buffer ) ); if ( write_len < 0 ) { if ( errno == EAGAIN ) { ++ poll_n; if ( block_time == 0 ) { block_time = time( NULL ); block_next = PIPE_BLOCK_INTERVAL; } else { block_secs = time( NULL ) - block_time; if ( block_secs > block_next ) { msg_eprintf( "pipe to logger blocked for %us\n", block_secs ); block_next += PIPE_BLOCK_INTERVAL; } } } else { msg_eprintf__exit( 1, "error in %s(): %s", "write", strerror( errno ) ); } } else { vrb_take( fifo_buffer, write_len ); try_again = 1; total_bytes = vrb_data_len( fifo_buffer ); update_totals(); if ( block_time != 0 ) { if ( time( NULL ) - block_time >= PIPE_BLOCK_INTERVAL ) { msg_eprintf( "pipe to logger block cleared\n" ); } block_time = 0; } } } //----------------------------------------------- // If nothing to write and input has ended, quit. //----------------------------------------------- else if ( read_eof ) _exit( 0 ); //------------------------------------------------- // If some I/O completed, try everything again now. //------------------------------------------------- if ( try_again ) continue; //-------------------------------------------- // If nothing to wait for, this is a deadlock. //-------------------------------------------- if ( poll_n == 0 ) { msg_eprintf__exit( 1, "poll event loop deadlock\n" ); } //--------------------------------------------- // Do poll for what events we need to wait for. //--------------------------------------------- poll_list[0].revents = 0; poll_list[1].revents = 0; errno = 0; if ( poll( poll_p, poll_n, FIFO_POLL_TIMEOUT ) < 0 ) { if ( errno == EINTR ) continue; msg_eprintf__exit( 1, "error in %s(): %s", "poll", strerror( errno ) ); } } } //----------------------------------------------------------------------------- // function clean_files // // purpose Do a cleanup of file descriptors. // // arguments 1 (int) number of descriptors required to close // // returns (unsigned int) number of descriptors actually closed //----------------------------------------------------------------------------- static unsigned int clean_files ( int arg_must_close ) { struct fd_node * this_node ; time_t time_now ; time_t limit_time ; unsigned int min_open ; unsigned int num_closed ; //---------------------------------------------------------- // Go through open files from oldest, deleting and closing // until the minimum number and the aging threshold are met. //---------------------------------------------------------- num_closed = 0; min_open = ( threshold * max_files ) / 10000; time_now = time( NULL ); for (;;) { if ( ! ( this_node = avl_first( & tree_by_time ) ) ) break; if ( arg_must_close <= 0 ) { if ( min_open < max_files && min_open < num_open ) { //-- Scale limit_time from idle_time to 0 based on how many open from min to max. limit_time = ( idle_time * ( max_files - num_open ) ) / ( idle_time * ( max_files - min_open ) ); } else { limit_time = idle_time; } if ( ( this_node->time.key + limit_time ) > time_now ) break; } else { -- arg_must_close; } avl_delete( & tree_by_name, this_node ); avl_delete( & tree_by_time, this_node ); close( this_node->fd ); ++ num_closed; -- num_open; -- total_files; free( this_node ); } return num_closed; } //----------------------------------------------------------------------------- // function get_open_file // // purpose Get a file descriptor for a specified file, using one already // open, or open a new one. // // arguments 1 (char *) the file name, modifiable in place. // // returns (int) file descriptor of open file // (int) -1 : error opening file // (int) -2 : ".." detected as a part of the name //----------------------------------------------------------------------------- int get_open_file ( char * arg_file_name ) { struct stat statbuf ; char * name_ptr ; char * file_name ; int dircount ; int owner ; int group ; int parstate ; int fd ; int statrc ; int tries ; //---------------------------------------------------- // Canonicalize file name by removing leading slashes. //---------------------------------------------------- file_name = arg_file_name; while ( * file_name == '/' ) ++ file_name; //-------------------------------------------------------------------- // Check directories from top down to get owner and group of last one. //-------------------------------------------------------------------- owner = 0; group = 0; dircount = 0; parstate = 0; name_ptr = file_name; for (;;) { if ( * name_ptr == 0 ) break; // end of name else if ( * name_ptr == '/' ) { if ( parstate == 2 ) return -2; // this means "/../" in the path * name_ptr = 0; // terminate directory name if ( lstat( file_name, & statbuf ) < 0 ) { setregid( group, group ); setreuid( 0, owner ); if ( mkdir( file_name, 0755 ) < 0 ) { fprintf( stderr, "mkdir( \"%s\", 0755 ): %s\n", file_name, strerror( errno ) ); } } else { // check for a symlink if ( S_ISLNK( statbuf.st_mode ) || ! ( S_ISDIR( statbuf.st_mode ) ) ) { * name_ptr = '/'; return -1; } owner = statbuf.st_uid; group = statbuf.st_gid; } * name_ptr = '/'; // put '/' back in name ++ dircount; parstate = 0; // indicate name start } else if ( parstate > 0 && * name_ptr == '.' ) ++ parstate; // remember leading '.' else { if ( * name_ptr <= 32 || * name_ptr > 126 ) * name_ptr = '_'; // valid characters only parstate = -1; // indicate not a directory } ++ name_ptr; } if ( parstate == 2 ) return -2; // last part is ".." //----------------------------------------------------------- // Try opening the file, cleaning file descriptors if needed. //----------------------------------------------------------- statrc = stat( file_name, & statbuf ); // note if file already exists. tries = 8; for (;;) { setregid( group, group ); setreuid( 0, owner ); fd = open( file_name, O_WRONLY | O_CREAT | O_APPEND, 0644 ); // try to open file if ( fd >= 0 ) { ++ total_opens; ++ total_files; return fd; // return good descriptor } fprintf( stderr, "open(\"%s\",,0644): %s\n", file_name, strerror( errno ) ); if ( -- tries == 0 ) return -1; // ran out of tries if ( errno != EMFILE && errno != ENFILE ) return -1; // cannot open anyway max_files = num_open; clean_files( 1 ); // clean some old files } } //----------------------------------------------------------------------------- // function process_log // // purpose Check for a log entry in the buffer and process any that are // are present. // // arguments -none- // // returns (size_t) length of data processed. //----------------------------------------------------------------------------- size_t process_log() { struct fd_node * this_node ; struct fd_name name_link ; char * name_ptr ; char * log_ptr ; char * take_end ; char * ptr ; size_t log_len ; size_t len ; int newline_tot ; int newline_seq ; int fd ; //------------------------------------------------------ // Make sure there is some data available in the buffer. //------------------------------------------------------ if ( ( len = vrb_data_len( read_buffer ) ) == 0 ) return 0; ptr = vrb_data_ptr( read_buffer ); //---------------------------------------------------------------- // Search for a non-newline to find the first line of a log entry. // The first line contains one or more file names. //---------------------------------------------------------------- for (;;) { if ( len == 0 ) return 0; if ( * ptr != '\n' ) break; -- len; ++ ptr; } name_ptr = ptr; //-------------------------------------------------------- // Search for a newline to find the end of the first line. // The log entry data begins immediately after it. //-------------------------------------------------------- for (;;) { if ( len == 0 ) return 0; if ( * ptr == '\n' ) break; -- len; ++ ptr; } -- len; ++ ptr; log_ptr = ptr; ++ total_lines; //--------------------------------------------------------------- // Search for a double newline to find the end of this log entry. // If the log entry data has multiple lines, then include both // newlines in the data. If not, include only one (but always // remove both from the buffer). //--------------------------------------------------------------- newline_tot = 0; newline_seq = 0; for (;;) { if ( len == 0 ) return 0; if ( * ptr == '\n' ) { ++ newline_tot; ++ newline_seq; ++ total_lines; } else newline_seq = 0; if ( newline_seq == 2 ) break; -- len; ++ ptr; } log_len = ptr - log_ptr; -- len; ++ ptr; take_end = ptr; if ( newline_tot > 2 ) log_len = ptr - log_ptr; //-------------------------------------- // For each file name in the first line // open the file and write the log data. //-------------------------------------- ptr = name_ptr; while ( ptr < log_ptr ) { //-- Get the next name from the first line and terminate it. name_ptr = ptr; while ( * ptr != '\n' && * ptr != NAME_SEP_CHAR ) ++ ptr; * ptr = 0; ++ ptr; //-- If the file is already open, get that file descriptor. name_link.key = name_ptr; if ( ( this_node = avl_find( & tree_by_name, & name_link, 0 ) ) ) { //-- Move the node to the front of the time tree. avl_delete( & tree_by_time, this_node ); this_node->time.key = time( NULL ); avl_insert_dup( & tree_by_time, this_node ); fd = this_node->fd; } //-- Else try to open the file. else { fd = get_open_file( name_ptr ); //-- Restore these. setreuid( ruid, euid ); setregid( rgid, egid ); //-- If it opened, add this fd to the collection. if ( fd >= 0 ) { this_node = malloc( sizeof (struct fd_node) + ( ptr - name_ptr ) ); if ( this_node ) { ++ num_open; strcpy( this_node->fname, name_ptr ); this_node->name.key = this_node->fname; this_node->time.key = time( NULL ); this_node->fd = fd; avl_insert( & tree_by_name, this_node ); avl_insert_dup( & tree_by_time, this_node ); } } } //-- If this file opened OK, write the log entry to it. if ( fd >= 0 ) { if ( write_block( fd, log_ptr, log_len ) < 0 ) { fprintf( stderr, "Error writing log entry to \"%s\": %s", name_ptr, strerror( errno ) ); } //-- If no node, then do not leave the fd open. if ( ! this_node ) { close( fd ); } } } //---------------------------- // Take used data from buffer. //---------------------------- len = take_end - vrb_data_ptr( read_buffer ); vrb_take( read_buffer, len ); ++ total_items; return len; } //----------------------------------------------------------------------------- // program weblogsplit // // function main // // purpose Split a log stream into individual files based on file names // integrated into the log format itself. // // usage Process started by the web server with the log stream piped // into stdin. // // syntax weblogsplit [-b buffersize] directory // // security If run as root, weblogsplit will change file owner:group to // match the directory it is in. To protect other files while // running as root, weblogsplit will call chroot() to change its // view of the root directory to the logging directory. // // format The log stream format must be modified to specify what file // each log entry is to be stored in. Each log entry must be // separated by two or more newline characters. The first line // that is not empty in an entry is the name (relative to the // logging directory) to store the entry in. The filename part // is not written but the rest of the entry, including the two // newlines, is written. //----------------------------------------------------------------------------- int main ( int argc , char * * argv ) { struct pollfd poll_one ; size_t log_len ; size_t page_size ; size_t fifo_bufsize ; size_t read_bufsize ; time_t time_now ; unsigned int time_next ; int errors ; int pipe_pid ; int time_remain ; int read_eof ; int pipe_fd [2]; int poll_n ; int flush_mode ; int dump_fd ; //------------------------------- // Set up message specifications. //------------------------------- argv_name = argv[0]; msg_set_program_name( argv_name ); msg_set_time_format( "%Y/%m/%d.%H:%M:%S." ); msg_set_frac_digits( 6 ); //------------------------------------------- // Briefly title the process as initializing. //------------------------------------------- argv_name[0] = 'I'; //---------------------------- // Do various initializations. //---------------------------- total_bytes = 0; total_lines = 0; total_items = 0; total_files = 0; total_opens = 0; num_open = 0; errors = 0; show_bytes_len = 0; show_lines_len = 0; show_items_len = 0; show_bytes_ptr = NULL; show_lines_ptr = NULL; show_items_ptr = NULL; //--------------------------------------------------------- // Get system page size according to the configured method. //--------------------------------------------------------- #ifdef _SC_PAGESIZE page_size = sysconf( _SC_PAGESIZE ); #else page_size = getpagesize(); #endif //------------------------------- // Set up configuration defaults. //------------------------------- fifo_bufsize = 0x00400000; read_bufsize = 4 * page_size; idle_time = 120; clean_cycle = 20; max_files = 256; threshold = 2500; // 2500 is 25% //---------------------------------------------------------- // Initialize two AVL trees to record open file descriptors. // One is indexed by file name and the other by last time. // Each node will be concurrently in both trees. //---------------------------------------------------------- avl_init( & tree_by_name, struct fd_node, name, fd_cmp_name ); avl_init( & tree_by_time, struct fd_node, time, fd_cmp_time ); //------------------------------------ // Scan options from the command line. //------------------------------------ while ( ( ++ argv, -- argc > 0 ) ) { //-------------------------- // Check for end of options. //-------------------------- if ( argv[0][0] != '-' ) { break; } else if ( argv[0][1] == '-' && argv[0][2] == 0 ) { ++ argv; -- argc; break; } //----------------------------------------------------------------------- // -B begins an option where in the command line the byte count is shown. // Additional characters in this option are used to reserve command line // space for the display of the byte count here in the process title. //----------------------------------------------------------------------- // NOTE: There being no standard for managing process title information // to be displayed via the "ps" command, this facility is thus dependent // on a specific implementation to work, in this case Linux. //----------------------------------------------------------------------- else if ( argv[0][1] == 'B' ) { show_bytes_ptr = argv[0]; show_bytes_len = strlen( show_bytes_ptr ); memset( show_bytes_ptr, 0, show_bytes_len ); } //----------------------------------------------------------------------- // -L begins an option where in the command line the line count is shown. // Additional characters in this option are used to reserve command line // space for the display of the line count here in the process title. //----------------------------------------------------------------------- // NOTE: There being no standard for managing process title information // to be displayed via the "ps" command, this facility is thus dependent // on a specific implementation to work, in this case Linux. //----------------------------------------------------------------------- else if ( argv[0][1] == 'L' ) { show_lines_ptr = argv[0]; show_lines_len = strlen( show_lines_ptr ); memset( show_lines_ptr, 0, show_lines_len ); } //----------------------------------------------------------------------- // -I begins an option where in the command line the item count is shown. // Additional characters in this option are used to reserve command line // space for the display of the item count here in the process title. //----------------------------------------------------------------------- // NOTE: There being no standard for managing process title information // to be displayed via the "ps" command, this facility is thus dependent // on a specific implementation to work, in this case Linux. //----------------------------------------------------------------------- else if ( argv[0][1] == 'I' ) { show_items_ptr = argv[0]; show_items_len = strlen( show_items_ptr ); memset( show_items_ptr, 0, show_items_len ); } //----------------------------------------------------------------------- // -F begins an option where in the command line the file count is shown. // Additional characters in this option are used to reserve command line // space for the display of the file count here in the process title. //----------------------------------------------------------------------- // NOTE: There being no standard for managing process title information // to be displayed via the "ps" command, this facility is thus dependent // on a specific implementation to work, in this case Linux. //----------------------------------------------------------------------- else if ( argv[0][1] == 'F' ) { show_files_ptr = argv[0]; show_files_len = strlen( show_files_ptr ); memset( show_files_ptr, 0, show_files_len ); } //----------------------------------------------------------------------- // -O begins an option where in the command line the open count is shown. // Additional characters in this option are used to reserve command line // space for the display of the open count here in the process title. //----------------------------------------------------------------------- // NOTE: There being no standard for managing process title information // to be displayed via the "ps" command, this facility is thus dependent // on a specific implementation to work, in this case Linux. //----------------------------------------------------------------------- else if ( argv[0][1] == 'O' ) { show_opens_ptr = argv[0]; show_opens_len = strlen( show_opens_ptr ); memset( show_opens_ptr, 0, show_opens_len ); } //------------------------------------- // -d is the directory to change to. // Otherwise use the current directory. //------------------------------------- else if ( str_find_arg( argv[0], "-d", "--dir", "--directory" ) ) { if ( ++ argv, ! -- argc || ! * argv ) { msg_eprintf__exit( 1, "directory name missing for option %s\n", argv[-1] ); } if ( chdir( argv[0] ) < 0 ) { msg_eprintf__exit( 1, "error changing to directory \"%s\": %s", argv[0], strerror( errno ) ); } } //------------------------------------------------------- // -f is the buffer size for the fifo queue process. // This size will be rounded up to a page size if needed. //------------------------------------------------------- else if ( str_find_arg( argv[0], "-f", "--fifobufsize", "--fifo-buf-size", "--fifobuffersize", "--fifo-buffer-size" ) ) { if ( ++ argv, ! -- argc || ! * argv ) { msg_eprintf__exit( 1, "buffer size missing for option %s\n", argv[-1] ); } fifo_bufsize = str_metric_to_ul( argv[0], NULL, 0, 1024, 1024 ); } //------------------------------------------------------- // -r is the buffer size for the reading process. // This size will be rounded up to a page size if needed. //------------------------------------------------------- else if ( str_find_arg( argv[0], "-r", "--readbufsize", "--read-buf-size", "--readbuffersize", "--read-buffer-size" ) ) { if ( ++ argv, ! -- argc || ! * argv ) { msg_eprintf__exit( 1, "buffer size missing for option %s\n", argv[-1] ); } read_bufsize = str_metric_to_ul( argv[0], NULL, 0, 1024, 1024 ); } //---------------------------------------------- // -i is the max idle file open time in seconds. // Files open longer than this time with nothing // written to them will be closed. //---------------------------------------------- else if ( str_find_arg( argv[0], "-i", "--idletime", "--idle-time" ) ) { if ( ++ argv, ! -- argc || ! * argv ) { msg_eprintf__exit( 1, "idle file open time missing for option %s\n", argv[-1] ); } idle_time = strtoul( argv[0], NULL, 0 ); } //------------------------------------------- // -c is the cleanup cycle period in seconds. //------------------------------------------- else if ( str_find_arg( argv[0], "-c", "--cleancycle", "--clean-cycle" ) ) { if ( ++ argv, ! -- argc || ! * argv ) { msg_eprintf__exit( 1, "cleanup cycle period time missing for option %s\n", argv[-1] ); } clean_cycle = strtoul( argv[0], NULL, 0 ); } //---------------------------------------------- // -m is the maximum number of split files open. //---------------------------------------------- else if ( str_find_arg( argv[0], "-m", "--maxfiles", "--max-files" ) ) { if ( ++ argv, ! -- argc || ! * argv ) { msg_eprintf__exit( 1, "maximum open file count missing for option %s\n", argv[-1] ); } max_files = strtoul( argv[0], NULL, 0 ); } //------------------------------------------ // -t is the threshold percentage for files. //------------------------------------------ else if ( str_find_arg( argv[0], "-t", "--threshold" ) ) { if ( ++ argv, ! -- argc || ! * argv ) { msg_eprintf__exit( 1, "file close threshold percentage missing for option %s\n", argv[-1] ); } threshold = (int) ( 10000.0 * ( 0.5 + str_metric_to_d( argv[0], NULL, 1.0, 1.0 ) ) ); } //----------------------- // Count unknown options. //----------------------- else { msg_eprintf( "unknown option: %s\n", argv[0] ); ++ errors; } } //---------------------------------------------------- // Additional arguments (not options) are not allowed. //---------------------------------------------------- while ( argc ) { msg_eprintf( "unknown argument \"%s\"\n", argv[0] ); -- argc; ++ argv; ++ errors; } //---------------------------- // Abort if there were errors. //---------------------------- if ( errors ) { msg_eprintf( "aborting due to %d error%s\n", errors, errors == 1 ? "" : "s" ); return 1; } //---------------------------------- // Adjust configuration into bounds. //---------------------------------- if ( threshold < 9 ) threshold = 9; if ( threshold > 9999 ) threshold = 9999; if ( max_files < 9 ) max_files = 9; if ( max_files > 9999 ) max_files = 9999; //-------------- // Know thyself. //-------------- ruid = getuid(); euid = geteuid(); rgid = getgid(); egid = getegid(); //---------------------------------------------------- // If running as root, then change the root directory. //---------------------------------------------------- if ( euid == 0 ) { if ( chroot( "." ) < 0 ) { msg_eprintf__exit( 1, "error changing root directory to current directory: %s", strerror( errno ) ); } } else if ( ruid == 0 ) { if ( seteuid( 0 ) < 0 ) { msg_eprintf__exit( 1, "error changing effective user id to root: %s", strerror( errno ) ); } if ( chroot( "." ) < 0 ) { msg_eprintf__exit( 1, "error changing root directory to current directory", strerror( errno ) ); } if ( seteuid( euid ) < 0 ) { msg_eprintf__exit( 1, "error changing effective user id to %d: %s", strerror( errno ) ); } } //----------------------------------------------------------- // If program is suid non-root, make sure we give up all root // permissions now by switching everything to the suid/euid. //----------------------------------------------------------- if ( egid != 0 ) setregid( egid, egid ); if ( euid != 0 ) setreuid( euid, euid ); //--------------------------------------------------------------------- // Set up a buffering process to pipe the data stream to this process. // This is to avoid blocking the web server writing to the log pipe if // there is any slowdown in the log splitting process. //--------------------------------------------------------------------- if ( pipe( pipe_fd ) < 0 ) { msg_eprintf__exit( 1, "error creating pipe for fifo buffering: %s", strerror( errno ) ); } fflush( stdout ); pipe_pid = fork(); if ( pipe_pid < 0 ) { msg_eprintf__exit( 1, "error forking process for fifo buffering: %s", strerror( errno ) ); } //------------------------------------------------- // Start up the child process to run a fifo buffer. //------------------------------------------------- if ( pipe_pid == 0 ) { argv_name[0] = 'B'; close( pipe_fd[0] ); if ( pipe_fd[1] != 1 ) { dup2( pipe_fd[1], 1 ); close( pipe_fd[1] ); } run_fifo_buffer( fifo_bufsize ); msg_eprintf__exit( 1, "buffering function returned with error: %s", strerror( errno ) ); } argv_name[0] = 'L'; close( pipe_fd[1] ); //-------------------------------------- // Initialize totals shown in proctitle. //-------------------------------------- update_totals(); //-------------------------------- // Move the pipe over to be stdin. //-------------------------------- if ( pipe_fd[0] != 0 ) { dup2( pipe_fd[0], 0 ); close( pipe_fd[0] ); } //----------------------------------------------------------- // Set descriptor non-blocking so we can use poll() to sleep. //----------------------------------------------------------- if ( fcntl( 0, F_SETFL, O_NONBLOCK ) < 0 ) { msg_eprintf__exit( 1, "error setting O_NONBLOCK on pipe input (fd 0): %s", strerror( errno ) ); } //-------------------------------------------------- // Set up a ring buffer to read the log stream into. //-------------------------------------------------- read_buffer = vrb_new( read_bufsize, NULL ); if ( ! read_buffer ) { msg_eprintf__exit( 1, "error creating buffer of %Zu bytes: %s", read_bufsize, strerror( errno ) ); } //--------------------------- // Start cleanup time period. //--------------------------- time_now = time( NULL ); time_next = time_now + clean_cycle + 1; time_next -= time_next % clean_cycle; //-------------------------------- // Initialize for poll event loop. //-------------------------------- poll_one.fd = 0; poll_one.events = POLLIN; read_eof = 0; flush_mode = 0; dump_fd = -1; //------------------------- // Do main processing loop. //------------------------- for (;;) { poll_n = 0; //-------------------------------------------------------------- // If buffer is full and cannot be reduced by processing, then // something has gone terribly wrong. Dump the buffer to a file // for later diagnosis and flush until the next double newline. //-------------------------------------------------------------- if ( vrb_space_len( read_buffer ) < page_size ) { char * this_end; char * write_ptr; size_t this_len; ssize_t write_len; this_len = vrb_data_len( read_buffer ); log_len = process_log(); if ( log_len == 0 || this_len == vrb_data_len( read_buffer ) ) { dump_fd = open( "/tmp/weblogsplit.dump", O_WRONLY | O_CREAT | O_EXCL | O_SYNC, 0644 ); if ( dump_fd >= 0 ) { write_ptr = vrb_data_ptr( read_buffer ); this_end = vrb_data_end( read_buffer ); while ( write_ptr < this_end ) { write_len = write( dump_fd, write_ptr, this_end - write_ptr ); if ( write_len <= 0 ) { close( dump_fd ); dump_fd = -1; break; } else { write_ptr += write_len; } } vrb_take( read_buffer, vrb_data_len( read_buffer ) ); } flush_mode = 2; } } //----------------------------------- // Try read if there is enough space. //----------------------------------- if ( ! read_eof && vrb_space_len( read_buffer ) >= page_size ) { ssize_t read_len; read_len = read( 0, vrb_space_ptr( read_buffer ), vrb_space_len( read_buffer ) ); if ( read_len < 0 ) { if ( errno == EAGAIN ) { ++ poll_n; } else { msg_eprintf__exit( 1, "error in %s(): %s", "read", strerror( errno ) ); } } else if ( read_len == 0 ) { close( 0 ); read_eof = 1; } else { vrb_give( read_buffer, read_len ); } } //------------------------------------------ // If data needs to be flushed, do that now. //------------------------------------------ if ( flush_mode > 0 ) { char * this_ptr; char * this_end; char * write_ptr; ssize_t write_len; this_ptr = vrb_data_ptr( read_buffer ); this_end = vrb_data_end( read_buffer ); while ( this_ptr < this_end ) { if ( * this_ptr == '\n' ) { if ( -- flush_mode <= 0 ) break; } else { flush_mode = 2; } ++ this_ptr; } write_ptr = vrb_data_ptr( read_buffer ); if ( dump_fd >= 0 ) { while ( write_ptr < this_ptr ) { write_len = write( dump_fd, write_ptr, this_ptr - write_ptr ); if ( write_len <= 0 ) { close( dump_fd ); dump_fd = -1; break; } else { write_ptr += write_len; } } if ( flush_mode <= 0 ) { close( dump_fd ); } } vrb_take( read_buffer, this_ptr - vrb_data_ptr( read_buffer ) ); } //------------------------------------------------------- // If there is any data in the buffer, try to process it. //------------------------------------------------------- if ( vrb_data_len( read_buffer ) > 0 ) { if ( ( log_len = process_log() ) > 0 ) { if ( show_bytes_ptr ) { total_bytes += log_len; update_totals(); // snprintf( show_bytes_ptr, show_bytes_len, "%llu", total_bytes ); } poll_n = 0; } } //--------------------------- // If end of input, quit now. //--------------------------- if ( read_eof && vrb_data_len( read_buffer ) == 0 ) _exit( 0 ); //------------------------------- // Check for periodic timed work. //------------------------------- if ( ( time_remain = time_next - time( NULL ) ) <= 0 ) { clean_files( 0 ); time_now = time( NULL ); time_next = time_now + clean_cycle + 1; time_next -= time_next % clean_cycle; time_remain = time_next - time_now; } //------------------- // Do poll if needed. //------------------- if ( poll_n ) { poll_one.revents = 0; errno = 0; if ( poll( & poll_one, 1, time_remain * 1000 ) < 0 ) { msg_eprintf( "error in %s()", "poll" ); } } } }