object implicits extends SQLImplicits with Serializable
(Scala-specific) Implicit methods available in Scala for converting
common Scala objects into DataFrame
s.
val sparkSession = SparkSession.builder.getOrCreate() import sparkSession.implicits._
- Since
2.0.0
- Alphabetic
- By Inheritance
- implicits
- Serializable
- Serializable
- SQLImplicits
- LowPrioritySQLImplicits
- AnyRef
- Any
- Hide All
- Show All
- Public
- All
Type Members
-
implicit
class
StringToColumn extends AnyRef
Converts $"col name" into a Column.
Converts $"col name" into a Column.
- Definition Classes
- SQLImplicits
- Since
2.0.0
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
_sqlContext: SQLContext
- Attributes
- protected
- Definition Classes
- implicits → SQLImplicits
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
equals(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
def
hashCode(): Int
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
implicit
def
localSeqToDatasetHolder[T](s: Seq[T])(implicit arg0: Encoder[T]): DatasetHolder[T]
Creates a Dataset from a local Seq.
Creates a Dataset from a local Seq.
- Definition Classes
- SQLImplicits
- Since
1.6.0
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
implicit
def
newBooleanArrayEncoder: Encoder[Array[Boolean]]
- Definition Classes
- SQLImplicits
- Since
1.6.1
-
implicit
def
newBooleanEncoder: Encoder[Boolean]
- Definition Classes
- SQLImplicits
- Since
1.6.0
-
implicit
def
newBoxedBooleanEncoder: Encoder[Boolean]
- Definition Classes
- SQLImplicits
- Since
2.0.0
-
implicit
def
newBoxedByteEncoder: Encoder[Byte]
- Definition Classes
- SQLImplicits
- Since
2.0.0
-
implicit
def
newBoxedDoubleEncoder: Encoder[Double]
- Definition Classes
- SQLImplicits
- Since
2.0.0
-
implicit
def
newBoxedFloatEncoder: Encoder[Float]
- Definition Classes
- SQLImplicits
- Since
2.0.0
-
implicit
def
newBoxedIntEncoder: Encoder[Integer]
- Definition Classes
- SQLImplicits
- Since
2.0.0
-
implicit
def
newBoxedLongEncoder: Encoder[Long]
- Definition Classes
- SQLImplicits
- Since
2.0.0
-
implicit
def
newBoxedShortEncoder: Encoder[Short]
- Definition Classes
- SQLImplicits
- Since
2.0.0
-
implicit
def
newByteArrayEncoder: Encoder[Array[Byte]]
- Definition Classes
- SQLImplicits
- Since
1.6.1
-
implicit
def
newByteEncoder: Encoder[Byte]
- Definition Classes
- SQLImplicits
- Since
1.6.0
-
implicit
def
newDateEncoder: Encoder[Date]
- Definition Classes
- SQLImplicits
- Since
2.2.0
-
implicit
def
newDoubleArrayEncoder: Encoder[Array[Double]]
- Definition Classes
- SQLImplicits
- Since
1.6.1
-
implicit
def
newDoubleEncoder: Encoder[Double]
- Definition Classes
- SQLImplicits
- Since
1.6.0
-
implicit
def
newDurationEncoder: Encoder[Duration]
- Definition Classes
- SQLImplicits
- Since
3.2.0
-
implicit
def
newFloatArrayEncoder: Encoder[Array[Float]]
- Definition Classes
- SQLImplicits
- Since
1.6.1
-
implicit
def
newFloatEncoder: Encoder[Float]
- Definition Classes
- SQLImplicits
- Since
1.6.0
-
implicit
def
newInstantEncoder: Encoder[Instant]
- Definition Classes
- SQLImplicits
- Since
3.0.0
-
implicit
def
newIntArrayEncoder: Encoder[Array[Int]]
- Definition Classes
- SQLImplicits
- Since
1.6.1
-
implicit
def
newIntEncoder: Encoder[Int]
- Definition Classes
- SQLImplicits
- Since
1.6.0
-
implicit
def
newJavaDecimalEncoder: Encoder[BigDecimal]
- Definition Classes
- SQLImplicits
- Since
2.2.0
-
implicit
def
newJavaEnumEncoder[A <: Enum[_]](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): Encoder[A]
- Definition Classes
- SQLImplicits
- Since
3.2.0
-
implicit
def
newLocalDateEncoder: Encoder[LocalDate]
- Definition Classes
- SQLImplicits
- Since
3.0.0
-
implicit
def
newLocalDateTimeEncoder: Encoder[LocalDateTime]
- Definition Classes
- SQLImplicits
- Since
3.4.0
-
implicit
def
newLongArrayEncoder: Encoder[Array[Long]]
- Definition Classes
- SQLImplicits
- Since
1.6.1
-
implicit
def
newLongEncoder: Encoder[Long]
- Definition Classes
- SQLImplicits
- Since
1.6.0
-
implicit
def
newMapEncoder[T <: Map[_, _]](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Encoder[T]
- Definition Classes
- SQLImplicits
- Since
2.3.0
-
implicit
def
newPeriodEncoder: Encoder[Period]
- Definition Classes
- SQLImplicits
- Since
3.2.0
-
implicit
def
newProductArrayEncoder[A <: Product](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): Encoder[Array[A]]
- Definition Classes
- SQLImplicits
- Since
1.6.1
-
implicit
def
newProductEncoder[T <: Product](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Encoder[T]
- Definition Classes
- LowPrioritySQLImplicits
- Since
1.6.0
-
implicit
def
newScalaDecimalEncoder: Encoder[BigDecimal]
- Definition Classes
- SQLImplicits
- Since
2.2.0
-
implicit
def
newSequenceEncoder[T <: Seq[_]](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Encoder[T]
- Definition Classes
- SQLImplicits
- Since
2.2.0
-
implicit
def
newSetEncoder[T <: Set[_]](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Encoder[T]
Notice that we serialize
Set
to Catalyst array.Notice that we serialize
Set
to Catalyst array. The set property is only kept when manipulating the domain objects. The serialization format doesn't keep the set property. When we have a Catalyst array which contains duplicated elements and convert it toDataset[Set[T]]
by using the encoder, the elements will be de-duplicated.- Definition Classes
- SQLImplicits
- Since
2.3.0
-
implicit
def
newShortArrayEncoder: Encoder[Array[Short]]
- Definition Classes
- SQLImplicits
- Since
1.6.1
-
implicit
def
newShortEncoder: Encoder[Short]
- Definition Classes
- SQLImplicits
- Since
1.6.0
-
implicit
def
newStringArrayEncoder: Encoder[Array[String]]
- Definition Classes
- SQLImplicits
- Since
1.6.1
-
implicit
def
newStringEncoder: Encoder[String]
- Definition Classes
- SQLImplicits
- Since
1.6.0
-
implicit
def
newTimeStampEncoder: Encoder[Timestamp]
- Definition Classes
- SQLImplicits
- Since
2.2.0
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
implicit
def
rddToDatasetHolder[T](rdd: RDD[T])(implicit arg0: Encoder[T]): DatasetHolder[T]
Creates a Dataset from an RDD.
Creates a Dataset from an RDD.
- Definition Classes
- SQLImplicits
- Since
1.6.0
-
implicit
def
symbolToColumn(s: Symbol): ColumnName
An implicit conversion that turns a Scala
Symbol
into a Column.An implicit conversion that turns a Scala
Symbol
into a Column.- Definition Classes
- SQLImplicits
- Since
1.3.0
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
def
toString(): String
- Definition Classes
- AnyRef → Any
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
Deprecated Value Members
-
def
newBooleanSeqEncoder: Encoder[Seq[Boolean]]
- Definition Classes
- SQLImplicits
- Deprecated
- Since
1.6.1
-
def
newByteSeqEncoder: Encoder[Seq[Byte]]
- Definition Classes
- SQLImplicits
- Deprecated
- Since
1.6.1
-
def
newDoubleSeqEncoder: Encoder[Seq[Double]]
- Definition Classes
- SQLImplicits
- Deprecated
- Since
1.6.1
-
def
newFloatSeqEncoder: Encoder[Seq[Float]]
- Definition Classes
- SQLImplicits
- Deprecated
- Since
1.6.1
-
def
newIntSeqEncoder: Encoder[Seq[Int]]
- Definition Classes
- SQLImplicits
- Deprecated
- Since
1.6.1
-
def
newLongSeqEncoder: Encoder[Seq[Long]]
- Definition Classes
- SQLImplicits
- Deprecated
- Since
1.6.1
-
def
newProductSeqEncoder[A <: Product](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): Encoder[Seq[A]]
- Definition Classes
- SQLImplicits
- Deprecated
- Since
1.6.1
-
def
newShortSeqEncoder: Encoder[Seq[Short]]
- Definition Classes
- SQLImplicits
- Deprecated
- Since
1.6.1
-
def
newStringSeqEncoder: Encoder[Seq[String]]
- Definition Classes
- SQLImplicits
- Deprecated
- Since
1.6.1