Search.setIndex({docnames:["index","pyspark","pyspark.ml","pyspark.mllib","pyspark.sql","pyspark.streaming"],envversion:{"sphinx.domains.c":1,"sphinx.domains.changeset":1,"sphinx.domains.cpp":1,"sphinx.domains.javascript":1,"sphinx.domains.math":2,"sphinx.domains.python":1,"sphinx.domains.rst":1,"sphinx.domains.std":1,"sphinx.ext.viewcode":1,sphinx:54},filenames:["index.rst","pyspark.rst","pyspark.ml.rst","pyspark.mllib.rst","pyspark.sql.rst","pyspark.streaming.rst"],objects:{"":{pyspark:[1,0,0,"-"]},"pyspark.Accumulator":{add:[1,2,1,""],value:[1,3,1,""]},"pyspark.AccumulatorParam":{addInPlace:[1,2,1,""],zero:[1,2,1,""]},"pyspark.BarrierTaskContext":{barrier:[1,2,1,""],get:[1,4,1,""],getTaskInfos:[1,2,1,""]},"pyspark.BasicProfiler":{profile:[1,2,1,""],stats:[1,2,1,""]},"pyspark.Broadcast":{destroy:[1,2,1,""],dump:[1,2,1,""],load:[1,2,1,""],load_from_path:[1,2,1,""],unpersist:[1,2,1,""],value:[1,3,1,""]},"pyspark.MarshalSerializer":{dumps:[1,2,1,""],loads:[1,2,1,""]},"pyspark.PickleSerializer":{dumps:[1,2,1,""],loads:[1,2,1,""]},"pyspark.Profiler":{dump:[1,2,1,""],profile:[1,2,1,""],show:[1,2,1,""],stats:[1,2,1,""]},"pyspark.RDD":{aggregate:[1,2,1,""],aggregateByKey:[1,2,1,""],barrier:[1,2,1,""],cache:[1,2,1,""],cartesian:[1,2,1,""],checkpoint:[1,2,1,""],coalesce:[1,2,1,""],cogroup:[1,2,1,""],collect:[1,2,1,""],collectAsMap:[1,2,1,""],combineByKey:[1,2,1,""],context:[1,3,1,""],count:[1,2,1,""],countApprox:[1,2,1,""],countApproxDistinct:[1,2,1,""],countByKey:[1,2,1,""],countByValue:[1,2,1,""],distinct:[1,2,1,""],filter:[1,2,1,""],first:[1,2,1,""],flatMap:[1,2,1,""],flatMapValues:[1,2,1,""],fold:[1,2,1,""],foldByKey:[1,2,1,""],foreach:[1,2,1,""],foreachPartition:[1,2,1,""],fullOuterJoin:[1,2,1,""],getCheckpointFile:[1,2,1,""],getNumPartitions:[1,2,1,""],getStorageLevel:[1,2,1,""],glom:[1,2,1,""],groupBy:[1,2,1,""],groupByKey:[1,2,1,""],groupWith:[1,2,1,""],histogram:[1,2,1,""],id:[1,2,1,""],intersection:[1,2,1,""],isCheckpointed:[1,2,1,""],isEmpty:[1,2,1,""],isLocallyCheckpointed:[1,2,1,""],join:[1,2,1,""],keyBy:[1,2,1,""],keys:[1,2,1,""],leftOuterJoin:[1,2,1,""],localCheckpoint:[1,2,1,""],lookup:[1,2,1,""],map:[1,2,1,""],mapPartitions:[1,2,1,""],mapPartitionsWithIndex:[1,2,1,""],mapPartitionsWithSplit:[1,2,1,""],mapValues:[1,2,1,""],max:[1,2,1,""],mean:[1,2,1,""],meanApprox:[1,2,1,""],min:[1,2,1,""],name:[1,2,1,""],partitionBy:[1,2,1,""],persist:[1,2,1,""],pipe:[1,2,1,""],randomSplit:[1,2,1,""],reduce:[1,2,1,""],reduceByKey:[1,2,1,""],reduceByKeyLocally:[1,2,1,""],repartition:[1,2,1,""],repartitionAndSortWithinPartitions:[1,2,1,""],rightOuterJoin:[1,2,1,""],sample:[1,2,1,""],sampleByKey:[1,2,1,""],sampleStdev:[1,2,1,""],sampleVariance:[1,2,1,""],saveAsHadoopDataset:[1,2,1,""],saveAsHadoopFile:[1,2,1,""],saveAsNewAPIHadoopDataset:[1,2,1,""],saveAsNewAPIHadoopFile:[1,2,1,""],saveAsPickleFile:[1,2,1,""],saveAsSequenceFile:[1,2,1,""],saveAsTextFile:[1,2,1,""],setName:[1,2,1,""],sortBy:[1,2,1,""],sortByKey:[1,2,1,""],stats:[1,2,1,""],stdev:[1,2,1,""],subtract:[1,2,1,""],subtractByKey:[1,2,1,""],sum:[1,2,1,""],sumApprox:[1,2,1,""],take:[1,2,1,""],takeOrdered:[1,2,1,""],takeSample:[1,2,1,""],toDebugString:[1,2,1,""],toLocalIterator:[1,2,1,""],top:[1,2,1,""],treeAggregate:[1,2,1,""],treeReduce:[1,2,1,""],union:[1,2,1,""],unpersist:[1,2,1,""],values:[1,2,1,""],variance:[1,2,1,""],zip:[1,2,1,""],zipWithIndex:[1,2,1,""],zipWithUniqueId:[1,2,1,""]},"pyspark.RDDBarrier":{mapPartitions:[1,2,1,""],mapPartitionsWithIndex:[1,2,1,""]},"pyspark.ResourceInformation":{addresses:[1,3,1,""],name:[1,3,1,""]},"pyspark.SparkConf":{contains:[1,2,1,""],get:[1,2,1,""],getAll:[1,2,1,""],set:[1,2,1,""],setAll:[1,2,1,""],setAppName:[1,2,1,""],setExecutorEnv:[1,2,1,""],setIfMissing:[1,2,1,""],setMaster:[1,2,1,""],setSparkHome:[1,2,1,""],toDebugString:[1,2,1,""]},"pyspark.SparkContext":{PACKAGE_EXTENSIONS:[1,3,1,""],accumulator:[1,2,1,""],addFile:[1,2,1,""],addPyFile:[1,2,1,""],applicationId:[1,3,1,""],binaryFiles:[1,2,1,""],binaryRecords:[1,2,1,""],broadcast:[1,2,1,""],cancelAllJobs:[1,2,1,""],cancelJobGroup:[1,2,1,""],defaultMinPartitions:[1,3,1,""],defaultParallelism:[1,3,1,""],dump_profiles:[1,2,1,""],emptyRDD:[1,2,1,""],getConf:[1,2,1,""],getLocalProperty:[1,2,1,""],getOrCreate:[1,4,1,""],hadoopFile:[1,2,1,""],hadoopRDD:[1,2,1,""],newAPIHadoopFile:[1,2,1,""],newAPIHadoopRDD:[1,2,1,""],parallelize:[1,2,1,""],pickleFile:[1,2,1,""],range:[1,2,1,""],resources:[1,3,1,""],runJob:[1,2,1,""],sequenceFile:[1,2,1,""],setCheckpointDir:[1,2,1,""],setJobDescription:[1,2,1,""],setJobGroup:[1,2,1,""],setLocalProperty:[1,2,1,""],setLogLevel:[1,2,1,""],setSystemProperty:[1,4,1,""],show_profiles:[1,2,1,""],sparkUser:[1,2,1,""],startTime:[1,3,1,""],statusTracker:[1,2,1,""],stop:[1,2,1,""],textFile:[1,2,1,""],uiWebUrl:[1,3,1,""],union:[1,2,1,""],version:[1,3,1,""],wholeTextFiles:[1,2,1,""]},"pyspark.SparkFiles":{get:[1,4,1,""],getRootDirectory:[1,4,1,""]},"pyspark.StatusTracker":{getActiveJobsIds:[1,2,1,""],getActiveStageIds:[1,2,1,""],getJobIdsForGroup:[1,2,1,""],getJobInfo:[1,2,1,""],getStageInfo:[1,2,1,""]},"pyspark.StorageLevel":{DISK_ONLY:[1,3,1,""],DISK_ONLY_2:[1,3,1,""],MEMORY_AND_DISK:[1,3,1,""],MEMORY_AND_DISK_2:[1,3,1,""],MEMORY_ONLY:[1,3,1,""],MEMORY_ONLY_2:[1,3,1,""],OFF_HEAP:[1,3,1,""]},"pyspark.TaskContext":{attemptNumber:[1,2,1,""],get:[1,4,1,""],getLocalProperty:[1,2,1,""],partitionId:[1,2,1,""],resources:[1,2,1,""],stageId:[1,2,1,""],taskAttemptId:[1,2,1,""]},"pyspark.sql":{Catalog:[4,1,1,""],CoGroupedData:[4,1,1,""],Column:[4,1,1,""],DataFrame:[4,1,1,""],DataFrameNaFunctions:[4,1,1,""],DataFrameReader:[4,1,1,""],DataFrameStatFunctions:[4,1,1,""],DataFrameWriter:[4,1,1,""],GroupedData:[4,1,1,""],Row:[4,1,1,""],SQLContext:[4,1,1,""],SparkSession:[4,1,1,""],UDFRegistration:[4,1,1,""],Window:[4,1,1,""],WindowSpec:[4,1,1,""],functions:[4,0,0,"-"],streaming:[4,0,0,"-"],types:[4,0,0,"-"]},"pyspark.sql.Catalog":{cacheTable:[4,2,1,""],clearCache:[4,2,1,""],createTable:[4,2,1,""],currentDatabase:[4,2,1,""],dropGlobalTempView:[4,2,1,""],dropTempView:[4,2,1,""],isCached:[4,2,1,""],listColumns:[4,2,1,""],listDatabases:[4,2,1,""],listFunctions:[4,2,1,""],listTables:[4,2,1,""],recoverPartitions:[4,2,1,""],refreshByPath:[4,2,1,""],refreshTable:[4,2,1,""],registerFunction:[4,2,1,""],setCurrentDatabase:[4,2,1,""],uncacheTable:[4,2,1,""]},"pyspark.sql.CoGroupedData":{apply:[4,2,1,""]},"pyspark.sql.Column":{alias:[4,2,1,""],asc:[4,2,1,""],asc_nulls_first:[4,2,1,""],asc_nulls_last:[4,2,1,""],astype:[4,2,1,""],between:[4,2,1,""],bitwiseAND:[4,2,1,""],bitwiseOR:[4,2,1,""],bitwiseXOR:[4,2,1,""],cast:[4,2,1,""],contains:[4,2,1,""],desc:[4,2,1,""],desc_nulls_first:[4,2,1,""],desc_nulls_last:[4,2,1,""],endswith:[4,2,1,""],eqNullSafe:[4,2,1,""],getField:[4,2,1,""],getItem:[4,2,1,""],isNotNull:[4,2,1,""],isNull:[4,2,1,""],isin:[4,2,1,""],like:[4,2,1,""],name:[4,2,1,""],otherwise:[4,2,1,""],over:[4,2,1,""],rlike:[4,2,1,""],startswith:[4,2,1,""],substr:[4,2,1,""],when:[4,2,1,""]},"pyspark.sql.DataFrame":{agg:[4,2,1,""],alias:[4,2,1,""],approxQuantile:[4,2,1,""],cache:[4,2,1,""],checkpoint:[4,2,1,""],coalesce:[4,2,1,""],colRegex:[4,2,1,""],collect:[4,2,1,""],columns:[4,3,1,""],corr:[4,2,1,""],count:[4,2,1,""],cov:[4,2,1,""],createGlobalTempView:[4,2,1,""],createOrReplaceGlobalTempView:[4,2,1,""],createOrReplaceTempView:[4,2,1,""],createTempView:[4,2,1,""],crossJoin:[4,2,1,""],crosstab:[4,2,1,""],cube:[4,2,1,""],describe:[4,2,1,""],distinct:[4,2,1,""],drop:[4,2,1,""],dropDuplicates:[4,2,1,""],drop_duplicates:[4,2,1,""],dropna:[4,2,1,""],dtypes:[4,3,1,""],exceptAll:[4,2,1,""],explain:[4,2,1,""],fillna:[4,2,1,""],filter:[4,2,1,""],first:[4,2,1,""],foreach:[4,2,1,""],foreachPartition:[4,2,1,""],freqItems:[4,2,1,""],groupBy:[4,2,1,""],groupby:[4,2,1,""],head:[4,2,1,""],hint:[4,2,1,""],intersect:[4,2,1,""],intersectAll:[4,2,1,""],isLocal:[4,2,1,""],isStreaming:[4,3,1,""],join:[4,2,1,""],limit:[4,2,1,""],localCheckpoint:[4,2,1,""],mapInPandas:[4,2,1,""],na:[4,3,1,""],orderBy:[4,2,1,""],persist:[4,2,1,""],printSchema:[4,2,1,""],randomSplit:[4,2,1,""],rdd:[4,3,1,""],repartition:[4,2,1,""],repartitionByRange:[4,2,1,""],replace:[4,2,1,""],rollup:[4,2,1,""],sample:[4,2,1,""],sampleBy:[4,2,1,""],schema:[4,3,1,""],select:[4,2,1,""],selectExpr:[4,2,1,""],show:[4,2,1,""],sort:[4,2,1,""],sortWithinPartitions:[4,2,1,""],stat:[4,3,1,""],storageLevel:[4,3,1,""],subtract:[4,2,1,""],summary:[4,2,1,""],take:[4,2,1,""],toDF:[4,2,1,""],toJSON:[4,2,1,""],toLocalIterator:[4,2,1,""],toPandas:[4,2,1,""],transform:[4,2,1,""],union:[4,2,1,""],unionAll:[4,2,1,""],unionByName:[4,2,1,""],unpersist:[4,2,1,""],where:[4,2,1,""],withColumn:[4,2,1,""],withColumnRenamed:[4,2,1,""],withWatermark:[4,2,1,""],write:[4,3,1,""],writeStream:[4,3,1,""]},"pyspark.sql.DataFrameNaFunctions":{drop:[4,2,1,""],fill:[4,2,1,""],replace:[4,2,1,""]},"pyspark.sql.DataFrameReader":{csv:[4,2,1,""],format:[4,2,1,""],jdbc:[4,2,1,""],json:[4,2,1,""],load:[4,2,1,""],option:[4,2,1,""],options:[4,2,1,""],orc:[4,2,1,""],parquet:[4,2,1,""],schema:[4,2,1,""],table:[4,2,1,""],text:[4,2,1,""]},"pyspark.sql.DataFrameStatFunctions":{approxQuantile:[4,2,1,""],corr:[4,2,1,""],cov:[4,2,1,""],crosstab:[4,2,1,""],freqItems:[4,2,1,""],sampleBy:[4,2,1,""]},"pyspark.sql.DataFrameWriter":{bucketBy:[4,2,1,""],csv:[4,2,1,""],format:[4,2,1,""],insertInto:[4,2,1,""],jdbc:[4,2,1,""],json:[4,2,1,""],mode:[4,2,1,""],option:[4,2,1,""],options:[4,2,1,""],orc:[4,2,1,""],parquet:[4,2,1,""],partitionBy:[4,2,1,""],save:[4,2,1,""],saveAsTable:[4,2,1,""],sortBy:[4,2,1,""],text:[4,2,1,""]},"pyspark.sql.GroupedData":{agg:[4,2,1,""],apply:[4,2,1,""],avg:[4,2,1,""],cogroup:[4,2,1,""],count:[4,2,1,""],max:[4,2,1,""],mean:[4,2,1,""],min:[4,2,1,""],pivot:[4,2,1,""],sum:[4,2,1,""]},"pyspark.sql.Row":{asDict:[4,2,1,""]},"pyspark.sql.SQLContext":{cacheTable:[4,2,1,""],clearCache:[4,2,1,""],createDataFrame:[4,2,1,""],dropTempTable:[4,2,1,""],getConf:[4,2,1,""],getOrCreate:[4,4,1,""],newSession:[4,2,1,""],range:[4,2,1,""],read:[4,3,1,""],readStream:[4,3,1,""],registerDataFrameAsTable:[4,2,1,""],registerFunction:[4,2,1,""],registerJavaFunction:[4,2,1,""],setConf:[4,2,1,""],sql:[4,2,1,""],streams:[4,3,1,""],table:[4,2,1,""],tableNames:[4,2,1,""],tables:[4,2,1,""],udf:[4,3,1,""],uncacheTable:[4,2,1,""]},"pyspark.sql.SparkSession":{Builder:[4,1,1,""],builder:[4,3,1,""],catalog:[4,3,1,""],conf:[4,3,1,""],createDataFrame:[4,2,1,""],getActiveSession:[4,4,1,""],newSession:[4,2,1,""],range:[4,2,1,""],read:[4,3,1,""],readStream:[4,3,1,""],sparkContext:[4,3,1,""],sql:[4,2,1,""],stop:[4,2,1,""],streams:[4,3,1,""],table:[4,2,1,""],udf:[4,3,1,""],version:[4,3,1,""]},"pyspark.sql.SparkSession.Builder":{appName:[4,2,1,""],config:[4,2,1,""],enableHiveSupport:[4,2,1,""],getOrCreate:[4,2,1,""],master:[4,2,1,""]},"pyspark.sql.UDFRegistration":{register:[4,2,1,""],registerJavaFunction:[4,2,1,""],registerJavaUDAF:[4,2,1,""]},"pyspark.sql.Window":{currentRow:[4,3,1,""],orderBy:[4,5,1,""],partitionBy:[4,5,1,""],rangeBetween:[4,5,1,""],rowsBetween:[4,5,1,""],unboundedFollowing:[4,3,1,""],unboundedPreceding:[4,3,1,""]},"pyspark.sql.WindowSpec":{orderBy:[4,2,1,""],partitionBy:[4,2,1,""],rangeBetween:[4,2,1,""],rowsBetween:[4,2,1,""]},"pyspark.sql.avro":{functions:[4,0,0,"-"]},"pyspark.sql.avro.functions":{from_avro:[4,6,1,""],to_avro:[4,6,1,""]},"pyspark.sql.functions":{PandasUDFType:[4,1,1,""],abs:[4,6,1,""],acos:[4,6,1,""],add_months:[4,6,1,""],approx_count_distinct:[4,6,1,""],array:[4,6,1,""],array_contains:[4,6,1,""],array_distinct:[4,6,1,""],array_except:[4,6,1,""],array_intersect:[4,6,1,""],array_join:[4,6,1,""],array_max:[4,6,1,""],array_min:[4,6,1,""],array_position:[4,6,1,""],array_remove:[4,6,1,""],array_repeat:[4,6,1,""],array_sort:[4,6,1,""],array_union:[4,6,1,""],arrays_overlap:[4,6,1,""],arrays_zip:[4,6,1,""],asc:[4,6,1,""],asc_nulls_first:[4,6,1,""],asc_nulls_last:[4,6,1,""],ascii:[4,6,1,""],asin:[4,6,1,""],atan2:[4,6,1,""],atan:[4,6,1,""],avg:[4,6,1,""],base64:[4,6,1,""],bin:[4,6,1,""],bitwiseNOT:[4,6,1,""],broadcast:[4,6,1,""],bround:[4,6,1,""],cbrt:[4,6,1,""],ceil:[4,6,1,""],coalesce:[4,6,1,""],col:[4,6,1,""],collect_list:[4,6,1,""],collect_set:[4,6,1,""],column:[4,6,1,""],concat:[4,6,1,""],concat_ws:[4,6,1,""],conv:[4,6,1,""],corr:[4,6,1,""],cos:[4,6,1,""],cosh:[4,6,1,""],count:[4,6,1,""],countDistinct:[4,6,1,""],covar_pop:[4,6,1,""],covar_samp:[4,6,1,""],crc32:[4,6,1,""],create_map:[4,6,1,""],cume_dist:[4,6,1,""],current_date:[4,6,1,""],current_timestamp:[4,6,1,""],date_add:[4,6,1,""],date_format:[4,6,1,""],date_sub:[4,6,1,""],date_trunc:[4,6,1,""],datediff:[4,6,1,""],dayofmonth:[4,6,1,""],dayofweek:[4,6,1,""],dayofyear:[4,6,1,""],decode:[4,6,1,""],degrees:[4,6,1,""],dense_rank:[4,6,1,""],desc:[4,6,1,""],desc_nulls_first:[4,6,1,""],desc_nulls_last:[4,6,1,""],element_at:[4,6,1,""],encode:[4,6,1,""],exp:[4,6,1,""],explode:[4,6,1,""],explode_outer:[4,6,1,""],expm1:[4,6,1,""],expr:[4,6,1,""],factorial:[4,6,1,""],first:[4,6,1,""],flatten:[4,6,1,""],floor:[4,6,1,""],format_number:[4,6,1,""],format_string:[4,6,1,""],from_csv:[4,6,1,""],from_json:[4,6,1,""],from_unixtime:[4,6,1,""],from_utc_timestamp:[4,6,1,""],get_json_object:[4,6,1,""],greatest:[4,6,1,""],grouping:[4,6,1,""],grouping_id:[4,6,1,""],hash:[4,6,1,""],hex:[4,6,1,""],hour:[4,6,1,""],hypot:[4,6,1,""],initcap:[4,6,1,""],input_file_name:[4,6,1,""],instr:[4,6,1,""],isnan:[4,6,1,""],isnull:[4,6,1,""],json_tuple:[4,6,1,""],kurtosis:[4,6,1,""],lag:[4,6,1,""],last:[4,6,1,""],last_day:[4,6,1,""],lead:[4,6,1,""],least:[4,6,1,""],length:[4,6,1,""],levenshtein:[4,6,1,""],lit:[4,6,1,""],locate:[4,6,1,""],log10:[4,6,1,""],log1p:[4,6,1,""],log2:[4,6,1,""],log:[4,6,1,""],lower:[4,6,1,""],lpad:[4,6,1,""],ltrim:[4,6,1,""],map_concat:[4,6,1,""],map_entries:[4,6,1,""],map_from_arrays:[4,6,1,""],map_from_entries:[4,6,1,""],map_keys:[4,6,1,""],map_values:[4,6,1,""],max:[4,6,1,""],md5:[4,6,1,""],mean:[4,6,1,""],min:[4,6,1,""],minute:[4,6,1,""],monotonically_increasing_id:[4,6,1,""],month:[4,6,1,""],months_between:[4,6,1,""],nanvl:[4,6,1,""],next_day:[4,6,1,""],ntile:[4,6,1,""],pandas_udf:[4,6,1,""],percent_rank:[4,6,1,""],posexplode:[4,6,1,""],posexplode_outer:[4,6,1,""],pow:[4,6,1,""],quarter:[4,6,1,""],radians:[4,6,1,""],rand:[4,6,1,""],randn:[4,6,1,""],rank:[4,6,1,""],regexp_extract:[4,6,1,""],regexp_replace:[4,6,1,""],repeat:[4,6,1,""],reverse:[4,6,1,""],rint:[4,6,1,""],round:[4,6,1,""],row_number:[4,6,1,""],rpad:[4,6,1,""],rtrim:[4,6,1,""],schema_of_csv:[4,6,1,""],schema_of_json:[4,6,1,""],second:[4,6,1,""],sequence:[4,6,1,""],sha1:[4,6,1,""],sha2:[4,6,1,""],shiftLeft:[4,6,1,""],shiftRight:[4,6,1,""],shiftRightUnsigned:[4,6,1,""],shuffle:[4,6,1,""],signum:[4,6,1,""],sin:[4,6,1,""],sinh:[4,6,1,""],size:[4,6,1,""],skewness:[4,6,1,""],slice:[4,6,1,""],sort_array:[4,6,1,""],soundex:[4,6,1,""],spark_partition_id:[4,6,1,""],split:[4,6,1,""],sqrt:[4,6,1,""],stddev:[4,6,1,""],stddev_pop:[4,6,1,""],stddev_samp:[4,6,1,""],struct:[4,6,1,""],substring:[4,6,1,""],substring_index:[4,6,1,""],sum:[4,6,1,""],sumDistinct:[4,6,1,""],tan:[4,6,1,""],tanh:[4,6,1,""],toDegrees:[4,6,1,""],toRadians:[4,6,1,""],to_csv:[4,6,1,""],to_date:[4,6,1,""],to_json:[4,6,1,""],to_str:[4,6,1,""],to_timestamp:[4,6,1,""],to_utc_timestamp:[4,6,1,""],translate:[4,6,1,""],trim:[4,6,1,""],trunc:[4,6,1,""],udf:[4,6,1,""],unbase64:[4,6,1,""],unhex:[4,6,1,""],unix_timestamp:[4,6,1,""],upper:[4,6,1,""],var_pop:[4,6,1,""],var_samp:[4,6,1,""],variance:[4,6,1,""],weekofyear:[4,6,1,""],when:[4,6,1,""],window:[4,6,1,""],xxhash64:[4,6,1,""],year:[4,6,1,""]},"pyspark.sql.functions.PandasUDFType":{COGROUPED_MAP:[4,3,1,""],GROUPED_AGG:[4,3,1,""],GROUPED_MAP:[4,3,1,""],MAP_ITER:[4,3,1,""],SCALAR:[4,3,1,""],SCALAR_ITER:[4,3,1,""]},"pyspark.sql.streaming":{DataStreamReader:[4,1,1,""],DataStreamWriter:[4,1,1,""],StreamingQuery:[4,1,1,""],StreamingQueryManager:[4,1,1,""]},"pyspark.sql.streaming.DataStreamReader":{csv:[4,2,1,""],format:[4,2,1,""],json:[4,2,1,""],load:[4,2,1,""],option:[4,2,1,""],options:[4,2,1,""],orc:[4,2,1,""],parquet:[4,2,1,""],schema:[4,2,1,""],text:[4,2,1,""]},"pyspark.sql.streaming.DataStreamWriter":{foreach:[4,2,1,""],foreachBatch:[4,2,1,""],format:[4,2,1,""],option:[4,2,1,""],options:[4,2,1,""],outputMode:[4,2,1,""],partitionBy:[4,2,1,""],queryName:[4,2,1,""],start:[4,2,1,""],trigger:[4,2,1,""]},"pyspark.sql.streaming.StreamingQuery":{awaitTermination:[4,2,1,""],exception:[4,2,1,""],explain:[4,2,1,""],id:[4,3,1,""],isActive:[4,3,1,""],lastProgress:[4,3,1,""],name:[4,3,1,""],processAllAvailable:[4,2,1,""],recentProgress:[4,3,1,""],runId:[4,3,1,""],status:[4,3,1,""],stop:[4,2,1,""]},"pyspark.sql.streaming.StreamingQueryManager":{active:[4,3,1,""],awaitAnyTermination:[4,2,1,""],get:[4,2,1,""],resetTerminated:[4,2,1,""]},"pyspark.sql.types":{ArrayType:[4,1,1,""],BinaryType:[4,1,1,""],BooleanType:[4,1,1,""],ByteType:[4,1,1,""],DataType:[4,1,1,""],DateType:[4,1,1,""],DecimalType:[4,1,1,""],DoubleType:[4,1,1,""],FloatType:[4,1,1,""],IntegerType:[4,1,1,""],LongType:[4,1,1,""],MapType:[4,1,1,""],NullType:[4,1,1,""],ShortType:[4,1,1,""],StringType:[4,1,1,""],StructField:[4,1,1,""],StructType:[4,1,1,""],TimestampType:[4,1,1,""]},"pyspark.sql.types.ArrayType":{fromInternal:[4,2,1,""],fromJson:[4,4,1,""],jsonValue:[4,2,1,""],needConversion:[4,2,1,""],simpleString:[4,2,1,""],toInternal:[4,2,1,""]},"pyspark.sql.types.ByteType":{simpleString:[4,2,1,""]},"pyspark.sql.types.DataType":{fromInternal:[4,2,1,""],json:[4,2,1,""],jsonValue:[4,2,1,""],needConversion:[4,2,1,""],simpleString:[4,2,1,""],toInternal:[4,2,1,""],typeName:[4,4,1,""]},"pyspark.sql.types.DateType":{EPOCH_ORDINAL:[4,3,1,""],fromInternal:[4,2,1,""],needConversion:[4,2,1,""],toInternal:[4,2,1,""]},"pyspark.sql.types.DecimalType":{jsonValue:[4,2,1,""],simpleString:[4,2,1,""]},"pyspark.sql.types.IntegerType":{simpleString:[4,2,1,""]},"pyspark.sql.types.LongType":{simpleString:[4,2,1,""]},"pyspark.sql.types.MapType":{fromInternal:[4,2,1,""],fromJson:[4,4,1,""],jsonValue:[4,2,1,""],needConversion:[4,2,1,""],simpleString:[4,2,1,""],toInternal:[4,2,1,""]},"pyspark.sql.types.ShortType":{simpleString:[4,2,1,""]},"pyspark.sql.types.StructField":{fromInternal:[4,2,1,""],fromJson:[4,4,1,""],jsonValue:[4,2,1,""],needConversion:[4,2,1,""],simpleString:[4,2,1,""],toInternal:[4,2,1,""],typeName:[4,2,1,""]},"pyspark.sql.types.StructType":{add:[4,2,1,""],fieldNames:[4,2,1,""],fromInternal:[4,2,1,""],fromJson:[4,4,1,""],jsonValue:[4,2,1,""],needConversion:[4,2,1,""],simpleString:[4,2,1,""],toInternal:[4,2,1,""]},"pyspark.sql.types.TimestampType":{fromInternal:[4,2,1,""],needConversion:[4,2,1,""],toInternal:[4,2,1,""]},"pyspark.streaming":{DStream:[5,1,1,""],StreamingContext:[5,1,1,""],StreamingListener:[5,1,1,""],kinesis:[5,0,0,"-"]},"pyspark.streaming.DStream":{cache:[5,2,1,""],checkpoint:[5,2,1,""],cogroup:[5,2,1,""],combineByKey:[5,2,1,""],context:[5,2,1,""],count:[5,2,1,""],countByValue:[5,2,1,""],countByValueAndWindow:[5,2,1,""],countByWindow:[5,2,1,""],filter:[5,2,1,""],flatMap:[5,2,1,""],flatMapValues:[5,2,1,""],foreachRDD:[5,2,1,""],fullOuterJoin:[5,2,1,""],glom:[5,2,1,""],groupByKey:[5,2,1,""],groupByKeyAndWindow:[5,2,1,""],join:[5,2,1,""],leftOuterJoin:[5,2,1,""],map:[5,2,1,""],mapPartitions:[5,2,1,""],mapPartitionsWithIndex:[5,2,1,""],mapValues:[5,2,1,""],partitionBy:[5,2,1,""],persist:[5,2,1,""],pprint:[5,2,1,""],reduce:[5,2,1,""],reduceByKey:[5,2,1,""],reduceByKeyAndWindow:[5,2,1,""],reduceByWindow:[5,2,1,""],repartition:[5,2,1,""],rightOuterJoin:[5,2,1,""],saveAsTextFiles:[5,2,1,""],slice:[5,2,1,""],transform:[5,2,1,""],transformWith:[5,2,1,""],union:[5,2,1,""],updateStateByKey:[5,2,1,""],window:[5,2,1,""]},"pyspark.streaming.StreamingContext":{addStreamingListener:[5,2,1,""],awaitTermination:[5,2,1,""],awaitTerminationOrTimeout:[5,2,1,""],binaryRecordsStream:[5,2,1,""],checkpoint:[5,2,1,""],getActive:[5,4,1,""],getActiveOrCreate:[5,4,1,""],getOrCreate:[5,4,1,""],queueStream:[5,2,1,""],remember:[5,2,1,""],socketTextStream:[5,2,1,""],sparkContext:[5,3,1,""],start:[5,2,1,""],stop:[5,2,1,""],textFileStream:[5,2,1,""],transform:[5,2,1,""],union:[5,2,1,""]},"pyspark.streaming.StreamingListener":{Java:[5,1,1,""],onBatchCompleted:[5,2,1,""],onBatchStarted:[5,2,1,""],onBatchSubmitted:[5,2,1,""],onOutputOperationCompleted:[5,2,1,""],onOutputOperationStarted:[5,2,1,""],onReceiverError:[5,2,1,""],onReceiverStarted:[5,2,1,""],onReceiverStopped:[5,2,1,""],onStreamingStarted:[5,2,1,""]},"pyspark.streaming.StreamingListener.Java":{"implements":[5,3,1,""]},"pyspark.streaming.kinesis":{InitialPositionInStream:[5,1,1,""],KinesisUtils:[5,1,1,""],utf8_decoder:[5,6,1,""]},"pyspark.streaming.kinesis.InitialPositionInStream":{LATEST:[5,3,1,""],TRIM_HORIZON:[5,3,1,""]},"pyspark.streaming.kinesis.KinesisUtils":{createStream:[5,5,1,""]},pyspark:{Accumulator:[1,1,1,""],AccumulatorParam:[1,1,1,""],BarrierTaskContext:[1,1,1,""],BarrierTaskInfo:[1,1,1,""],BasicProfiler:[1,1,1,""],Broadcast:[1,1,1,""],MarshalSerializer:[1,1,1,""],PickleSerializer:[1,1,1,""],Profiler:[1,1,1,""],RDD:[1,1,1,""],RDDBarrier:[1,1,1,""],ResourceInformation:[1,1,1,""],SparkConf:[1,1,1,""],SparkContext:[1,1,1,""],SparkFiles:[1,1,1,""],SparkJobInfo:[1,1,1,""],SparkStageInfo:[1,1,1,""],StatusTracker:[1,1,1,""],StorageLevel:[1,1,1,""],TaskContext:[1,1,1,""],sql:[4,0,0,"-"],streaming:[5,0,0,"-"]}},objnames:{"0":["py","module","Python module"],"1":["py","class","Python class"],"2":["py","method","Python method"],"3":["py","attribute","Python attribute"],"4":["py","classmethod","Python class method"],"5":["py","staticmethod","Python static method"],"6":["py","function","Python function"]},objtypes:{"0":"py:module","1":"py:class","2":"py:method","3":"py:attribute","4":"py:classmethod","5":"py:staticmethod","6":"py:function"},terms:{"13t06":4,"16be":4,"16le":4,"1a2s3a":4,"1e4":4,"1e6":4,"32le":4,"3bc51062973c458d5a6f2d8d64a023246354ad7e064b1e4e009ec8a0699a3043":4,"3c01bdbb26f358bab27f267924aa2c9a03fcfdb8":4,"902fbdd2b1df0c4f70b4a5d23525e932":4,"9223372036854775807l":4,"9223372036854775808l":4,"abstract":[0,1,5],"boolean":4,"byte":[1,4,5],"case":[1,4],"char":4,"class":[1,4,5],"default":[1,4,5],"enum":4,"final":[1,4],"float":[1,4],"function":[0,1,5],"import":[1,4],"int":[1,4],"long":[1,4,5],"new":[1,4,5],"null":[1,4,5],"public":1,"return":[1,4,5],"short":4,"static":[1,4,5],"switch":1,"throw":[4,5],"true":[1,4,5],"try":[1,4],"while":[1,4,5],AND:4,AWS:5,Added:[1,4],And:1,For:[1,4],IDs:4,Its:[1,4],LQS:4,NOT:[1,4],Not:1,One:[1,4],STS:5,THEN:4,That:4,The:[1,4,5],There:4,These:[1,4],Use:[1,4],Used:1,Uses:4,Using:[1,4],With:[1,4],_c0:4,_c1:4,_c2:4,_jconf:1,_jvm:1,a_map:4,aaaac:4,ababab:4,abc:4,abcd123:4,abcd:4,abort:1,about:[1,4],abov:4,abs:[1,4],absenc:5,absolut:[1,4],accept:4,access:[1,4],accesskeyid:5,accord:[1,4],accordingli:4,account:4,accum_param:1,accumul:1,accumulatorparam:1,accuraci:[1,4],achiev:4,aco:4,acquir:1,across:[1,4],action:1,activ:[1,4,5],actual:[1,4],add:[1,4,5],add_month:4,add_on:4,added:[1,4],addfil:1,adding:[4,5],addinplac:1,addit:[1,4],addition:4,addpyfil:1,address:1,addstreaminglisten:5,advis:1,affect:1,after:[1,4,5],again:[1,4],against:[4,5],age2:4,age3:4,age:4,agecol:4,ages:4,agg:4,aggreg:[1,4],aggregatebykei:1,agnost:4,aid:1,aka:4,algorithm:[1,4],ali:4,alia:[1,4],alias:4,alic:4,align:4,all:[1,4,5],alloc:1,allow:[1,4,5],allowbackslashescapinganycharact:4,allowcom:4,allowloc:1,allownumericleadingzero:4,allowsinglequot:4,allowunquotedcontrolchar:4,allowunquotedfieldnam:4,along:1,alreadi:[1,4],also:[1,4],alter:4,altern:4,although:[1,4],alwai:[1,4,5],amazonaw:5,america:4,among:4,amount:4,an_arrai:4,analysi:4,analysisexcept:4,analyz:4,angl:4,ani:[1,4],anint:4,anoth:[1,4,5],answer:4,anti:4,apach:[1,4,5],api:[1,4,5],app:1,appear:4,append:[1,4],appl:1,appli:[1,4,5],applic:[1,4,5],application_1433865536131_34483:1,applicationid:1,appnam:[1,4],approx_count_distinct:4,approxim:[1,4],approxquantil:4,arbitrari:[1,4],arg1:4,arg2:4,arg:[1,4],argument:[1,4,5],arn:5,around:4,arr:4,arrai:[1,4],array_contain:4,array_distinct:4,array_except:4,array_intersect:4,array_join:4,array_max:4,array_min:4,array_posit:4,array_remov:4,array_repeat:4,array_sort:4,array_union:4,arrays_overlap:4,arrays_zip:4,arraytyp:4,arriv:4,arrow:4,art:1,asc:4,asc_nulls_first:4,asc_nulls_last:4,ascend:[1,4],ascii:4,asdict:4,asin:4,asnondeterminist:4,asof_join:4,assign:[1,4],associ:[1,4,5],assum:[1,4,5],assumpt:4,astyp:4,atan2:4,atan:4,atom:4,attempt:[1,4],attemptnumb:1,attribut:4,autobatchedseri:1,automat:[1,4],avail:[1,4],averag:[1,4],avg:4,avoid:[1,4],avro:0,avrodf:4,awaitanytermin:4,awaittermin:[4,5],awaitterminationortimeout:5,awar:4,awsaccesskeyid:5,awssecretkei:5,axi:4,back:4,background:4,backslash:4,backward:4,bad:1,banana:1,bar:[1,4],barrier:1,barriertaskcontext:1,barriertaskinfo:1,base64:4,base:[1,4,5],basic:[0,1,4,5],basicprofil:1,batch:[1,4,5],batch_df:4,batch_id:4,batch_it:4,batchcomplet:5,batchdur:5,batchid:4,batchsiz:1,batchstart:5,batchsubmit:5,bcp:4,becaus:4,becom:4,been:[1,4,5],befor:[1,4,5],begin:[4,5],behav:1,behavior:4,behind:4,being:[1,4,5],belong:[1,4],below:4,benefit:4,better:1,between:[1,4,5],bewar:4,beyond:4,bia:1,big:4,bigint:4,billion:4,bin:4,binari:[1,4,5],binaryfil:1,binaryrecord:1,binaryrecordsstream:5,binarytyp:4,bit:4,bitwis:4,bitwiseand:4,bitwisenot:4,bitwiseor:4,bitwisexor:4,blanklin:4,block:[1,4],bob:4,bool:4,booleantyp:4,both:[1,4,5],bound:4,boundari:4,box:1,branch:1,broadcast:[1,4],brotli:4,bround:4,bucket:[1,4],bucketbi:4,bucketed_t:4,build:[1,4],builder:4,built:[1,4],builtin:4,bytearrai:4,bytebuff:1,bytetyp:4,bzip2:4,cach:[1,4,5],cachedexecutoridletimeout:1,cachet:4,calcul:[4,5],call:[1,4,5],caller:4,came:4,can:[1,4,5],cancel:1,cancelalljob:1,canceljobgroup:1,cannot:[1,4],captur:1,cardin:1,carri:1,cartesian:[1,4],casesensit:4,cast:4,cast_all_to_int:4,cat:1,catalog:4,catalyst:4,categori:4,caus:[1,4],caution:1,cbrt:4,cd9fb1e148ccd8442e5aa74904cc73bf6fb54d1d54d333bd596aa9bb4bb4e961:4,cdf:4,ceil:4,cell:4,certain:4,chain:[1,4],chang:[1,4,5],charact:4,character:5,charset:4,chartoescapequoteescap:4,check:[1,4],checkcod:1,checkpoint:[1,4,5],checkpointinterv:5,checkpointloc:4,checkpointpath:5,choos:1,chosen:1,circuit:4,claim:4,classif:0,classmethod:[1,4,5],classnam:1,claus:4,clean:4,clear:[1,4],clearcach:4,client:5,clone:1,close:[1,4],closest:4,cloudwatch:5,club:4,cluster:[0,1,4,5],coalesc:[1,4],code:[1,4],codec:[1,4],coeffici:4,cogroup:[1,4,5],cogrouped_map:4,cogroupeddata:4,col0:4,col1:4,col1_:4,col2:4,col:4,col_nam:4,collect:[0,1,4,5],collect_list:4,collect_set:4,collectasmap:1,colnam:4,colregex:4,column:[0,4],columnnameofcorruptrecord:4,columnprun:4,com:5,combfunc:1,combin:[1,4],combinebykei:[1,5],combop:1,come:4,command:1,comment:4,commit:4,common:4,commonli:1,commut:[1,5],compar:1,compat:[1,4,5],competit:4,complet:[1,4,5],compon:4,compress:[1,4],compressioncodecclass:1,comput:[1,4,5],concat:4,concat_w:4,concaten:4,concis:4,concret:4,concurr:1,cond:4,condit:4,conf:[1,4],confid:1,config:4,configur:[1,4],conflict:4,confus:4,connect:[1,4,5],consecut:4,consid:[1,4],consist:[1,4],consol:4,constant:1,constraint:4,construct:4,consum:[1,4],contain:[1,4,5],containsnul:4,content:[0,4],context:[0,1,5],contextu:1,conting:4,continu:[4,5],contrast:4,control:[1,4],conv:4,convers:4,convert:[1,4],coordin:[4,5],copi:[1,4,5],core:[4,5],corr:4,correct:[1,4],correctli:4,correl:4,correspond:[4,5],corrupt:4,cos:4,cosh:4,cosin:4,cost:4,could:[1,4,5],count:[1,4,5],countapprox:1,countapproxdistinct:1,countbykei:1,countbyvalu:[1,5],countbyvalueandwindow:5,countbywindow:5,countdistinct:4,counter:1,counterpart:4,countri:4,cours:4,cov:4,covar_pop:4,covar_samp:4,covari:4,cover:4,cprofil:1,crash:4,crc32:4,creat:[1,4,5],create_map:4,createcombin:[1,5],createdatafram:4,createglobaltempview:4,createorreplaceglobaltempview:4,createorreplacetempview:4,createstream:5,createt:4,createtempview:4,creation:4,credenti:5,cross:4,crossjoin:4,crosstab:4,csv:4,csv_sdf:4,ctx:1,cube:4,cume_dist:4,cumul:4,current:[1,4,5],current_d:4,current_timestamp:4,currentdatabas:4,currentrow:4,custom:[1,4],cyclic:4,dai:4,data:[0,1,4,5],data_typ:4,databas:4,datafram:[0,1,4],dataframenafunct:4,dataframeread:4,dataframestatfunct:4,dataframewrit:4,dataset:[0,1,4],datasourc:4,datastreamread:4,datastreamwrit:4,datatyp:[1,4],date1:4,date2:4,date:4,date_add:4,date_format:4,date_sub:4,date_trunc:4,datediff:4,dateformat:4,datetim:[4,5],datetimeformatt:4,datetyp:4,dayofmonth:4,dayofweek:4,dayofyear:4,dbname:4,ddl:4,dead:1,deal:4,debug:[1,4],decid:4,decim:4,decimaltyp:4,decod:[1,4,5],decreas:[1,5],dedupl:4,deeper:4,def:[1,4],defaultawscredentialsproviderchain:5,defaultminpartit:1,defaultparallel:1,defaultvalu:[1,4],defin:[1,4,5],deflat:4,degre:4,delai:4,delaythreshold:4,delet:1,delim:4,delimit:[4,5],dense_rank:4,depart:4,depend:[1,4,5],deploi:4,deploy:4,deprec:[1,4],depth:1,deptid:4,desc:4,desc_nulls_first:4,desc_nulls_last:4,descend:[1,4],describ:[1,4],descript:1,deseri:[1,4],design:1,desir:4,destroi:1,detail:[1,4,5],detect:4,determin:4,determinist:4,develop:5,developerapi:1,deviat:[1,4],df0:4,df1:4,df2:4,df3:4,df4:4,df5:4,df_as1:4,df_as2:4,diamond:4,dict:[1,4],dictionari:[1,4],did:[1,4],diff:4,differ:[1,4,5],digest:4,digit:4,dimens:1,dimension:4,directli:[1,4],directori:[1,4,5],dirnam:1,dirpath:1,disabl:4,discoveri:4,discret:[0,5],discuss:4,disk:[1,4],disk_onli:1,disk_only_2:[1,4],displai:4,distanc:4,distinct:[1,4,5],distinct_ag:4,distribut:[0,1,4],divid:1,doc:1,doctest:[1,4],document:[4,5],doe:[1,4,5],doesn:4,doi:4,doing:4,domain:4,don:[1,4],done:[1,4,5],dot:4,dotnet:4,doubl:4,doubletyp:4,down:1,download:1,drastic:4,driver:[1,4],drop:[1,4],drop_dupl:4,dropdupl:4,dropfieldifallnul:4,dropglobaltempview:4,dropmalform:4,dropna:4,droptempt:4,droptempview:4,dsl:4,dstream:[0,5],dtype:4,due:[1,4],dump:1,dump_profil:1,duplic:[1,4],durat:[4,5],dure:[1,4,5],dynam:1,dynamicalloc:1,dynamodb:5,each:[1,4,5],eager:4,earli:4,earlier:5,earn:4,east:5,edf:4,effect:1,effici:[1,4,5],egg:1,either:[1,4,5],elaps:5,element:[1,4,5],element_at:4,elementtyp:4,elimin:[4,5],ellipsi:1,els:4,embed:4,emit:4,empti:[1,4],emptyrdd:1,emptyvalu:4,enabl:[4,5],enablehivesupport:4,enclos:4,encod:[1,4,5],end:[1,4,5],endpointurl:5,endswith:4,enforceschema:4,engin:1,enough:4,ensur:[1,4,5],enter:5,entir:[1,4],entri:[0,1,4,5],env:1,environ:1,ephemer:1,epoch:[1,4],epoch_id:4,epoch_ordin:4,epochid:4,eqnullsaf:4,equal:4,equi:4,equival:[1,4,5],err:4,error:[1,4,5],errorifexist:4,errorornul:4,escap:4,escapequot:4,especi:4,estim:[1,4],etc:[1,4,5],evalu:[0,4],even:[1,4],evenli:[1,4],event:[4,5],eventtim:4,ever:4,everi:[1,4,5],everyth:4,evolv:[1,4],exact:4,exactli:[1,4],exampl:[1,4],except:[1,4,5],exceptal:4,exclus:[1,4],execut:[1,4,5],executor:[1,4],exist:[1,4,5],existingrdd:4,exp:4,expand:4,expect:[1,4],expens:[1,4],experiment:[1,4],expir:5,explain:4,explicit:4,explicitli:4,explod:4,explode_out:4,exploratori:4,expm1:4,exponenti:4,expos:1,expr:4,express:4,extend:[1,4],extern:[1,4,5],extra:[1,4],extract:4,face:4,factori:4,fail:[1,4],failfast:4,failur:[1,4],fair:1,fall:1,fallback:1,fals:[1,4,5],famili:4,fast:[1,4],faster:1,fatal:1,fault:[1,5],featur:[0,1],feed:4,fetch:[1,4],few:5,fewer:[1,4],field1:4,field2:4,field:4,fieldnam:4,fifth:4,file1:1,file2:1,file:[1,4,5],fileinput:1,filenam:1,filesystem:1,filev:1,fill:4,fillna:4,filter:[1,4,5],filter_func:4,filterfunc:5,find:[1,4],finer:1,finish:1,first:[1,4,5],firstli:4,fit:4,five:4,fix:[1,4,5],flag:[1,4],flat:[1,5],flatmap:[1,5],flatmapvalu:[1,5],flatten:[1,4,5],fleec:1,float64:4,floattyp:4,floor:4,fold:1,foldbykei:1,follow:[1,4],foo:[1,4],forc:1,forcibl:4,foreach:[1,4],foreachbatch:4,foreachpartit:[1,4],foreachrdd:5,forev:4,forget:4,fork:1,form:[1,4],format:[1,4],format_numb:4,format_str:4,former:1,found:[1,4],fpga:1,fpm:0,fraction:[1,4],frame:4,freqitem:4,frequenc:4,frequent:4,fresh:4,fri:4,from:[1,4,5],from_arrow_typ:4,from_avro:4,from_csv:4,from_json:4,from_unixtim:4,from_utc_timestamp:4,frombas:4,fromintern:4,fromjson:4,ftp:1,full:[4,5],full_out:4,fulli:[1,4],fullout:4,fullouterjoin:[1,5],func:[1,4,5],functiontyp:4,further:4,futur:[1,4],gap:[1,4],garbag:[1,5],gatewai:1,gd1:4,gd2:4,gdf:4,gender:4,gener:[1,4,5],get:[1,4,5],get_json_object:4,getact:5,getactivejobsid:1,getactiveorcr:5,getactivesess:4,getactivestageid:1,getal:1,getcheckpointfil:1,getconf:[1,4],getfield:4,getitem:4,getjobidsforgroup:1,getjobinfo:1,getlocalproperti:1,getnumpartit:[1,4],getoffset:4,getorcr:[1,4,5],getrootdirectori:1,getstageinfo:1,getstoragelevel:1,gettaskinfo:1,give:[1,4],given:[1,4,5],glob:[1,4],global:[1,4],global_temp:4,globfilt:4,glom:[1,5],gmt:4,goe:4,going:[1,4],gpu:1,gracefulli:5,grain:1,grape:1,graph:5,graphx:1,greater:[1,4],greatest:4,greenwald:4,group:[0,1,4],groupbi:[1,4],groupbykei:[1,5],groupbykeyandwindow:5,grouped_agg:4,grouped_map:4,groupeddata:4,groupid:1,grouping_id:4,groupwith:1,grow:4,guarante:[1,4],guid:4,gzip:4,gzipcodec:1,had:[1,4],hadoop:[1,4,5],hadoopfil:1,hadooprdd:1,half_even:4,half_up:4,handl:[1,4],hang:1,happen:4,hard:4,hardcod:4,has:[1,4,5],hash:[1,4,5],have:[1,4,5],hdf:[1,5],head:4,header:4,heart:4,height:4,hello:[1,4],help:1,helper:1,henc:4,here:[1,4],hex:4,hexadecim:4,high:1,hint:4,histogram:1,hit:1,hive:4,hold:[1,4],hook_compress:1,host:1,hostnam:5,hour:[4,5],hourli:4,how:[1,4,5],howev:[1,4],html:1,http:[1,4,5],human:1,hyperbol:4,hyperloglog:1,hypot:4,iam:5,ice:4,ident:4,identifi:[1,4,5],ids:[1,4],idx:4,ietf:4,iff:4,ignor:[1,4,5],ignore_exception_detail:4,ignoreleadingwhitespac:4,ignorenul:4,ignorenullfield:4,ignoretrailingwhitespac:4,imag:0,immedi:[1,4],immut:1,implement:[1,4,5],implicitli:4,includ:[1,4,5],inclus:4,incomplet:1,incorpor:4,incorrect:4,increas:[1,4,5],increment:[1,4,5],independ:4,indeterminist:4,index:[1,4],indic:[1,4],individu:1,inf:4,infer:[1,4],inferschema:4,infin:[1,4],info:[1,5],inform:[1,4],inherit:1,initcap:4,initi:[1,4,5],initialpositioninstream:5,initialrdd:5,inner:4,input:[1,4,5],input_df:4,input_file_nam:4,inputformat:1,inputformatclass:1,insensit:4,inser:1,insert:4,insertinto:4,insid:[1,4],instal:[1,4],instanc:[1,4],instanti:[1,4],instead:[1,4],instr:4,int32:4,int64:4,integ:[1,4],integertyp:4,integr:4,intend:4,intent:5,intention:1,interfac:[1,4],intermedi:4,intern:[1,4,5],interpret:[1,4,5],interrupt:1,interruptoncancel:1,intersect:[1,4],intersectal:4,interv:[4,5],intlist:4,introduc:4,intwrit:1,invalid:4,invers:[4,5],invert:5,invfunc:5,invoc:[1,4],invok:[1,4],invreducefunc:5,ipv4:1,irrecover:1,irrespect:4,is_cach:1,isact:4,iscach:4,ischeckpoint:1,isempti:1,isin:4,isloc:4,islocallycheckpoint:1,isn:4,isnan:4,isnotnul:4,isnul:4,iso:4,isstream:4,issu:4,istemporari:4,item:[1,4],iter:[1,4],its:[1,4,5],itself:4,jar:1,java:[1,4,5],javaclassnam:4,javardd:5,javastreamingcontext:5,javastringlength2:4,javastringlength3:4,javastringlength:4,javatowritableconvert:1,javaudaf:4,jdbc:4,jdf:4,jdstream:5,jgd:4,job:[1,5],job_to_cancel:1,jobgroup:1,jobid:1,john:4,join:[1,4,5],joined_df:4,jrdd:1,jrdd_deseri:[1,5],jsc:1,json:4,json_sdf:4,json_tupl:4,jsonformatschema:4,jsonvalu:4,jsparksess:4,jspec:4,jsq:4,jsqlcontext:4,jsqm:4,jssc:5,jst:4,jstring:4,jtracker:1,just:4,jvm:[1,4],karp:4,kcl:5,keep:[1,4],keepseri:5,kei:[1,4,5],kept:1,key1:4,key2:4,keybi:1,keyclass:1,keyconvert:1,keyfunc:1,keytyp:4,keyword:4,khanna:4,kind:4,kinesi:0,kinesisappnam:5,kinesisutil:5,kitten:4,know:4,known:[1,4],krap:4,kth:1,kurtosi:4,kwarg:4,label:4,lag:4,laid:4,lamb:1,lambda:[1,4],lang:4,languag:[1,4],larg:[1,4,5],large_broadcast:1,larger:4,largest:[1,4],last:[1,4,5],last_dai:4,lastprogress:4,late:4,later:4,latest:[4,5],latter:[1,4],launch:1,layer:1,lead:4,learn:4,leas:5,least:[1,4],leav:4,left:[1,4,5],left_anti:4,left_out:4,left_semi:4,leftanti:4,leftout:4,leftouterjoin:[1,5],leftsemi:4,lemon:1,len:[1,4],length:[1,4,5],less:4,letter:4,level:[1,4,5],levenshtein:4,lib:1,librari:[1,5],lifecycl:4,lifetim:4,like:[1,4],limit:[1,4,5],linalg:0,line:[1,4,5],lineag:1,linesep:4,link:1,list:[1,4,5],listcolumn:4,listdatabas:4,listfunct:4,listtabl:4,lit:4,liter:4,littl:1,live:5,load:[1,4,5],load_from_path:1,loaddefault:1,local:[1,4],local_tim:4,localcheckpoint:[1,4],locat:[1,4,5],lock:1,log10:4,log1p:4,log2:4,log:[1,4],logarithm:4,logic:4,loglevel:1,longer:[1,4],longtyp:4,longwrit:1,look:4,lookup:1,loop:4,los_angel:4,low:1,lower:[1,4],lowerbound:4,lpad:4,ltrim:4,lz4:4,lzo:4,machin:4,made:[1,4],mai:[1,4],main:[0,1,4,5],make:[4,5],malform:4,mallori:4,manag:4,mani:[1,4],manipul:4,manner:[1,4],map1:4,map2:4,map3:4,map:[1,4,5],map_concat:4,map_entri:4,map_from_arrai:4,map_from_entri:4,map_func:1,map_it:4,map_kei:4,map_valu:4,mapfield:4,mapinpanda:4,mappartit:[1,4,5],mappartitionswithindex:[1,5],mappartitionswithsplit:1,mapper:1,mapr:1,mapreduc:1,maptyp:4,mapvalu:[1,5],mari:1,mark:[1,4],marshal:1,marshalseri:1,master:[1,4,5],match:[1,4],materi:1,math:4,mathemat:4,matter:1,max:[1,4],maxcharspercolumn:4,maxcolumn:4,maximum:[1,4],maxmalformedlogperpartit:4,maxrecordsperbatch:4,maxsiz:4,md5:4,mean:[1,4],mean_udf:4,mean_v:4,meanapprox:1,meant:4,measur:4,mechan:[1,4],median:4,meet:[1,4],memori:[1,4],memory_and_disk:[1,4],memory_and_disk_2:[1,5],memory_onli:[1,5],memory_only_2:1,merg:[1,4],merge_asof:4,mergecombin:[1,5],mergeschema:4,mergevalu:[1,5],messag:5,metadata:[1,4],metastor:4,method:[1,4,5],metric:5,micro:4,microsecond:4,middl:4,might:[1,4],mike:4,millisecond:4,min:[1,4],min_udf:4,minim:4,minimum:[1,4,5],minpartit:1,minsplit:1,minu:4,minut:4,mismatch:4,miss:[1,4],mix:4,mkdir:1,mkdtemp:4,mllib:[0,1],mode:[1,4],model:4,modif:4,modifi:1,modul:[0,1],moment:4,mon:4,monitor:[1,5],monoton:4,monotonically_increasing_id:4,month:4,months_between:4,more:[1,4,5],most:[1,4],move:5,mpi:1,mpi_barri:1,much:[1,4],multi:[1,4],multilin:4,multipl:[1,4,5],multiply_two_col:4,multiply_two_nested_col:4,must:[1,4,5],mutat:[1,4],my_tabl:4,mycustomprofil:1,mydoubleavg:4,mypassword:4,myrdd:1,nalic:4,name:[0,1,4,5],namedtemporaryfil:1,namedtupl:4,namespac:4,nan:4,nanvalu:4,nanvl:4,narrow:4,nativ:4,natur:4,nbar:1,nearli:1,need:[1,4],needconvers:4,neg:4,negativeinf:4,nest:4,neutral:1,new_random_udf:4,newapihadoopfil:1,newapihadooprdd:1,newli:4,newlin:4,newsess:4,next:[1,4],next_dai:4,next_dat:4,next_month:4,nfoo:1,nnnnn:1,node:[1,4],non:[1,4],nondeterminist:4,none:[1,4,5],noqa:4,normal:[1,4],note:4,now:4,nthi:4,ntile:4,null_replac:4,nullabl:4,nulltyp:4,nullvalu:4,num:[1,4,5],number:[1,4,5],numbit:4,numbucket:4,numer:[1,4],numpartit:[1,4,5],numpi:4,numrecentprogressupd:4,numslic:1,obj:[1,4],object:[1,4,5],obtain:1,occurr:4,off:[1,4],off_heap:1,offset:4,often:[1,4],old:[1,5],older:4,omit:4,onbatchcomplet:5,onbatchstart:5,onbatchsubmit:5,onc:[1,4,5],one:[1,4,5],oneatatim:5,oneatwobthreec:4,onli:[1,4,5],onlin:4,onoutputoperationcomplet:5,onoutputoperationstart:5,onreceivererror:5,onreceiverstart:5,onreceiverstop:5,onstreamingstart:5,oom:4,open:[1,4],openhook:1,oper:[1,4,5],opt1:4,opt2:4,opt3:4,optim:4,option:[1,4,5],orc:4,orc_df:4,orc_partit:4,orc_sdf:4,order:[1,4,5],orderbi:4,ordereddict:4,ordin:4,org:[1,4,5],origin:[1,4],other:[1,4,5],otherwis:[1,4],our:1,out:[1,4],outer:[1,4,5],output:[1,4],outputformat:1,outputformatclass:1,outputmod:4,outputoperationcomplet:5,outputoperationstart:5,outsid:5,over:[1,4,5],overflow:4,overlap:4,overload:4,overrid:[1,4],overwrit:4,own:4,p362:4,packag:0,package_extens:1,pad:4,page:0,pair:[1,4,5],panda:4,pandas_udf:4,pandasudftyp:4,papadimitri:4,parallel:[1,4,5],param:[0,1,4],paramet:[1,4,5],parent:[1,5],parquet:4,parquet_partit:4,parquet_sdf:4,pars:4,parser:4,part:[1,4],partial:4,particular:1,partit:[1,4,5],partition:[1,5],partition_id:4,partitionbi:[1,4,5],partitionfunc:[1,5],partitionid:[1,4],pass:[1,4],password:4,past:4,path:[1,4],pathglobfilt:4,pattern:[1,4],pdf:4,pdf_iter:4,pearson:4,peopl:4,people1:4,per:[1,4,5],percent_rank:4,percentag:4,percentil:4,perform:[1,4],period:[1,4,5],permiss:4,permut:4,persist:[1,4,5],person:4,peter:4,physic:4,pick:5,pickl:1,pickle_registri:1,picklefil:1,pickleseri:1,pid:4,pipe:1,pipelin:0,pivot:4,pivot_col:4,place:[1,4],plan:[1,4],pleas:[1,4],plu:4,plus_i:4,plus_on:4,point:[0,1,4,5],polar:4,polici:5,pool:1,popul:4,port:[1,5],portable_hash:[1,5],pos:4,posexplod:4,posexplode_out:4,posit:[4,5],positiveinf:4,possibl:[1,4],potenti:[1,4],pow:4,power:4,pprint:5,practic:[1,4],pre:[1,4],preced:4,precis:4,predic:[1,4,5],prefer:1,prefersdecim:4,prefetch:[1,4],prefetchpartit:[1,4],prefix:5,prepar:1,present:4,preserv:4,preservespartit:[1,5],pretti:4,prev_dat:4,previou:[4,5],previous:1,primit:[1,4],primitivesasstr:4,print:[1,4,5],print_row:4,printabl:1,printf:4,printschema:4,prior:4,prioriti:1,probabl:[1,4],process:[1,4,5],processallavail:4,processingtim:4,produc:[1,4],product:[1,4],profil:1,profiler_cl:1,program:[1,4,5],programm:1,progress:[1,4],project:4,propag:[1,4],properti:[1,4,5],propos:4,provid:[1,4,5],prune:4,pst:4,pstat:1,pull:5,purpos:[1,4],put:4,py4jjavaerror:4,pyfil:1,pyrolit:1,pyspark:0,python:[1,4],pythonstreaminglisten:5,qualifi:[1,4],quantil:4,quarter:4,quartil:4,queri:[4,5],query_explain:4,querynam:4,queue:5,queuestream:5,quot:4,quotat:4,quoteal:4,radian:4,rais:[1,4],rand:4,randint:4,randn:4,random:[0,1,4],random_udf:4,randomli:[1,4],randomsplit:[1,4],rang:[1,4],rangebetween:4,rangeexchang:4,rangefram:4,rank:4,rate:1,rather:[1,4],ratio:4,rdd1:1,rdd2:1,rdd:[0,1,4,5],rdd_a:5,rdd_b:5,rddbarrier:1,reach:1,read:[1,4,5],readabl:1,readi:4,readlin:1,readstream:4,real:4,reason:4,receiv:[1,5],receivererror:5,receiverstart:5,receiverstop:5,recent:[1,4],recentprogress:4,recogn:5,recommend:[0,1,4],recomput:1,record:[1,4,5],recordlength:[1,5],recov:4,recoverpartit:4,recreat:5,recurs:[1,4],redistribut:1,reduc:[1,5],reducebykei:[1,5],reducebykeyandwindow:5,reducebykeyloc:1,reducebywindow:5,reducefunc:5,reduct:5,redund:4,refer:1,reflect:4,refresh:4,refreshbypath:4,refresht:4,regener:4,regex:4,regexp:4,regexp_extract:4,regexp_replac:4,region:5,regionnam:5,regist:[1,4],registerdataframeast:4,registerfunct:4,registerjavafunct:4,registerjavaudaf:4,registr:4,regress:0,regular:4,rel:[1,4],relat:[1,4,5],relativeerror:4,relativesd:1,relaunch:1,releas:[1,4,5],relev:4,reli:4,reliabl:[1,4,5],rememb:5,remov:[1,4],renam:4,render:4,rep:4,repartit:[1,4,5],repartitionandsortwithinpartit:1,repartitionbyrang:4,repeat:4,replac:[1,4],replic:1,report:[1,5],repres:[1,4,5],represent:[1,4,5],reprocess:4,request:4,requir:[1,4],reseri:1,resettermin:4,resili:[0,1],resolv:[1,4],resourc:[1,4],resourceinform:1,respect:[4,5],respond:1,respons:4,restart:[1,4],result:[1,4,5],retain:[1,4,5],retainedjob:1,retainedstag:1,retriev:4,returntyp:4,reus:1,revers:4,right:[1,4,5],right_out:4,rightout:4,rightouterjoin:[1,5],rint:4,risk:4,rlike:4,rnlt:4,role:5,rollup:4,root:[1,4],roughli:1,round:4,roundoff:4,routin:1,row1:4,row2:4,row3:4,row:4,row_numb:4,rowfram:4,rowprint:4,rowsbetween:4,rpad:4,rsd:4,rtrim:4,rule:4,run:[1,4,5],runid:4,runjob:1,runtim:4,sacrific:1,safe:[1,4],sai:4,salari:4,sale:4,same:[1,4,5],sampl:[1,4],samplebi:4,samplebykei:1,samplesizeperpartit:4,samplestdev:1,samplevari:1,samplingratio:4,sat:4,satisfi:[1,5],save:[1,4,5],saveashadoopdataset:1,saveashadoopfil:1,saveasnewapihadoopdataset:1,saveasnewapihadoopfil:1,saveaspicklefil:1,saveassequencefil:1,saveast:4,saveastextfil:[1,5],savemod:4,scala:[1,4],scalar:4,scalar_it:4,scale:4,scan:[1,4],scenario:1,schedul:[1,4,5],schema:4,schema_of_csv:4,schema_of_json:4,scheme:4,schenker:4,sdf:4,sdf_schema:4,search:[0,1,4],second:[1,4,5],secretkei:5,section:4,secur:5,see:[1,4,5],seed:[1,4],seen:[4,5],select:4,selectexpr:4,self:[1,4],semant:[1,4],semi:4,send:1,sensit:4,sent:1,sentenc:4,sep:4,separ:4,seqfunc:1,seqop:1,sequenc:[1,4,5],sequencefil:1,sequencefileoutputformat:1,sequenti:[1,4],sequncefil:1,serd:4,seri:4,serial:[1,4],serializ:4,servic:5,session:[4,5],set:[1,4,5],setal:1,setappnam:1,setcheckpointdir:[1,4],setconf:4,setcurrentdatabas:4,setexecutorenv:1,setifmiss:1,setjobdescript:1,setjobgroup:1,setlocalproperti:1,setloglevel:1,setmast:1,setnam:1,setsparkhom:1,setsystemproperti:1,setter:1,setup:5,setupfunc:5,sever:4,sha1:4,sha2:4,sha:4,shall:4,share:[1,4],shell:1,shift:4,shiftleft:4,shiftright:4,shiftrightunsign:4,ship:1,shorten:4,shorthand:4,shorttyp:4,should:[1,4,5],show:[1,4],show_profil:1,shown:4,shuffl:[1,4],shut:1,side:[1,4],sign:4,signifi:4,signum:4,silent:4,similar:[1,4,5],similarli:1,simpl:4,simplestr:4,simpli:4,sin:4,sinc:[1,4],sine:4,singl:[1,4,5],singleton:1,sinh:4,sink:4,sit:4,size:[1,4],skew:4,skip:[1,4],sleep:1,slen:4,slice:[4,5],slide:[4,5],slidedur:[4,5],slow:1,slower:5,small:[1,4],smaller:1,snappi:4,sock_fil:1,socket:5,sockettextstream:5,some:[1,4],someth:1,somewhat:1,sort:[1,4],sort_arrai:4,sort_columns_asc:4,sortbi:[1,4],sortbykei:1,sorted_bucketed_t:4,sortwithinpartit:4,soundex:4,sourc:[1,4,5],space:[1,4],spade:4,span:4,spark:[1,4,5],spark_partition_id:4,spark_us:[1,4],sparkconf:[1,4],sparkcontext:[0,1,4,5],sparkexcept:1,sparkfil:1,sparkhom:1,sparkjobinfo:1,sparksess:[0,4],sparkstageinfo:1,sparkui:1,sparkus:1,special:[1,4],specif:[1,4],specifi:[1,4,5],speed:4,spip:1,split:[1,4],split_expand:4,splitindex:1,spy:4,sql:[0,1],sql_ctx:4,sqlconf:4,sqlcontext:4,sqlqueri:4,sqm:4,sqrt:4,squar:4,srccol:4,ssc:5,sssxxx:4,stackoverflowexcept:4,stage:1,stageid:1,stai:4,stamp:4,standalon:4,standard:[1,4],start:[1,4,5],start_job:1,startpo:4,startswith:4,starttim:[1,4],stat:[0,1,4],statcount:1,state:[1,4,5],statement:4,statist:4,statu:[1,4],statustrack:1,std:4,stddev:4,stddev_pop:4,stddev_samp:4,stdev:1,stdout:1,step:[1,4],still:4,stop:[1,4,5],stop_job:1,stopgracefulli:5,stopsparkcontext:5,storag:[1,4,5],storagelevel:[1,4,5],store:[4,5],str:[1,4],strata:4,stratifi:[1,4],stratum:4,stream:[0,1],streaming_queri:4,streamingcontext:[0,5],streaminglisten:5,streamingqueri:4,streamingqueryexcept:4,streamingquerymanag:4,streamingqueryprogress:4,streamingstart:5,streamlib:1,streamnam:5,stride:4,string:[1,4,5],stringlengthint:4,stringlengthstr:4,stringtyp:4,strlen:4,strongli:[1,4],struct1:4,struct2:4,struct:4,structfield:4,structtyp:4,structur:4,stsassumerolearn:5,stsexternalid:5,stssessionnam:5,style:4,submit:[1,5],subnam:4,subpackag:0,subprotocol:4,subsequ:[1,4],subset:[1,4],substr:4,substring_index:4,subsystem:4,subtract:[1,4,5],subtractbykei:1,successfulli:4,suffix:5,suggest:1,suit:4,suitabl:4,sum:[1,4],sum_udf:4,sumapprox:1,sumdistinct:4,summari:4,sun:4,support:[1,4],suppress:1,sure:5,symbol:4,synchron:4,syntax:4,sys:4,system:[1,4,5],tab:4,tabl:4,table1:4,tablenam:4,tag:4,take:[1,4],takeord:1,takesampl:1,tan:4,tangent:4,tanh:4,target:[1,4],task:[1,4],taskattemptid:1,taskcontext:1,tbl:4,tcp:5,temp:4,tempdir:1,tempfil:[1,4],tempfile2:1,tempfile3:1,temporari:4,temptablealreadyexistsexcept:4,ten:4,term:1,termin:[4,5],test:[1,4],test_support:4,testfil:1,text:[1,4,5],text_sdf:4,textfil:[1,4],textfilestream:5,textinputformat:1,than:[1,4,5],that_queri:4,thei:[1,4],them:[1,4,5],therefor:4,theta:4,thi:[1,4,5],thin:4,third:4,this_queri:4,those:1,though:4,thread:[1,4,5],three:[1,4,5],thresh:4,through:[1,4],thrown:4,thu:[1,4],tie:4,tied:4,ties:4,time:[1,4,5],time_df:4,timecolumn:4,timeout:[1,4,5],timestamp:4,timestampformat:4,timestamptyp:4,timezon:4,tinyint:4,tip:5,tmp2:1,tmp:1,tmpfile:1,tmptabl:4,to_arrow_typ:4,to_avro:4,to_csv:4,to_dat:4,to_json:4,to_list:1,to_profil:1,to_replac:4,to_str:4,to_timestamp:4,to_upp:4,to_utc_timestamp:4,tobas:4,todebugstr:1,todegre:4,todf:4,togeth:[1,4],tointern:4,tointernald:4,tojson:4,token:4,toler:[1,5],tolocaliter:[1,4],tom:4,too:4,tool:1,top:[1,4],topanda:4,toplevelrecord:4,toradian:4,tostr:1,total:[1,4],trace:1,traceback:[1,4],track:[1,4],trail:4,transact:4,transaction:4,transform:[4,5],transformfunc:5,transformwith:5,translat:[1,4],treat:4,tree:[0,1,4],treeaggreg:1,treereduc:1,tri:4,trigger:[1,4],trim:4,trim_horizon:5,trunc:4,truncat:[1,4],trust:5,tue:4,tumbl:4,tune:0,tup:4,tupl:[1,4],turn:[1,4],two:[1,4,5],twobthreec:4,txt:[1,4],type:[0,1,5],typenam:4,u0000:4,u612:4,udf:4,udfregistr:4,uhrbach:4,uiweburl:1,unbase64:4,unbias:4,unbound:4,unboundedfollow:4,unboundedpreced:4,uncach:4,uncachet:4,uncompress:4,undefin:4,under:[1,4],underflow:4,underli:4,understand:1,unhex:4,unicod:[1,4,5],unifi:5,union:[1,4,5],unional:4,unionbynam:4,uniqu:[1,4,5],unit:[1,4],unix:4,unix_tim:4,unix_timestamp:[4,5],unknown:4,unless:4,unlik:4,unlimit:4,unmatch:4,unnam:4,unnecessari:4,unord:4,unpars:4,unpersist:[1,4],unquot:4,unset:4,unsign:4,unsupport:4,until:[1,4],updat:[1,4,5],updatefunc:5,updatestatebykei:5,upper:4,upperbound:4,upstream:[1,4],uri:1,url:[1,4,5],usabl:1,usag:[1,4],use:[1,4,5],use_unicod:[1,4],used:[1,4,5],usedisk:1,useful:[1,4],usememori:1,useoffheap:1,user:[1,4],uses:[1,4,5],using:[1,4,5],utc:4,utc_tim:4,utf8:5,utf8_decod:5,utf:[1,4,5],util:[0,4],uuuu:4,val:4,valid:[1,4,5],vals1:4,vals2:4,valu:[1,4,5],value12:4,value1:[1,4],value2:[1,4],valueclass:1,valuecontainsnul:4,valueconvert:1,valueerror:1,valuetyp:4,var_pop:4,var_samp:4,vari:1,variabl:1,varianc:[1,4],variant:4,variat:4,variou:[1,4,5],vector:[1,4],veri:[1,4],verifi:4,verifyschema:4,version:[1,4],vertic:4,via:[1,4],view:4,viewnam:4,wai:[1,4],wait:[1,4,5],want:[4,5],warn:1,watermark:4,weak:1,web:[1,4],wed:4,week:4,weekofyear:4,weight:[1,4],well:1,were:4,what:1,whatev:4,when:[1,4,5],where:[1,4,5],wherea:4,whether:[1,4],which:[1,4,5],white:1,whitespac:4,who:1,whole:4,wholetext:4,wholetextfil:1,whose:[1,4],width:[4,5],window:[4,5],windowdur:[4,5],windowspec:4,wise:4,wish:[1,5],withcolumn:4,withcolumnrenam:4,within:[1,4,5],without:[1,4,5],withreplac:[1,4],withwatermark:4,won:1,word:4,work:[1,4],workaround:4,worker:[1,5],world:1,would:[1,4],wrap:[1,4],wrapper:4,writabl:1,write:[1,4],writer:4,writestream:4,written:[1,4,5],wrong_kei:4,wrriten:5,x00:4,x02:4,x04:4,x0cspade:4,xor:4,xrang:1,xxhash64:4,xxhash:4,y_bc:4,yarn:1,year:4,yes:4,yet:[1,4],yield:[1,4],you:[1,4,5],your:[1,4,5],yyi:4,yyyi:4,zero:[1,4],zerovalu:1,zip:[1,4],zipwithindex:1,zipwithuniqueid:1,zlib:4,zone:4,zstd:4},titles:["Welcome to Spark Python API Docs!","pyspark package","pyspark.ml package","pyspark.mllib package","pyspark.sql module","pyspark.streaming module"],titleterms:{"class":0,"function":4,api:[0,2],avro:4,classif:[2,3],cluster:[2,3],content:[1,5],context:4,core:0,distribut:3,doc:0,evalu:[2,3],featur:[2,3],fpm:[2,3],imag:2,indic:0,kinesi:5,linalg:[2,3],mllib:3,modul:[2,3,4,5],packag:[1,2,3],param:2,pipelin:2,pyspark:[1,2,3,4,5],python:0,random:3,recommend:[2,3],regress:[2,3],spark:0,sql:4,stat:[2,3],stream:[4,5],subpackag:1,tabl:0,tree:3,tune:2,type:4,util:[2,3],welcom:0}})
